From bc2d734e9a842ffcdd0508cdd47d578247066576 Mon Sep 17 00:00:00 2001
From: Eric Wasylishen <ewasylishen@gmail.com>
Date: Wed, 8 Feb 2017 02:08:29 -0700
Subject: [PATCH] light: move Lightmap_Soften into WriteLightmaps

---
 include/light/light.hh |  2 +-
 light/light.cc         |  2 +-
 light/ltface.cc        | 66 +++++-------------------------------------
 3 files changed, 10 insertions(+), 60 deletions(-)

diff --git a/include/light/light.hh b/include/light/light.hh
index f5f73ef6..95bb4372 100644
--- a/include/light/light.hh
+++ b/include/light/light.hh
@@ -190,7 +190,7 @@ extern int numDirtVectors;
 extern bool dirt_in_use;               // should any dirtmapping take place? set in SetupDirt
 
 extern float fadegate;
-extern int softsamples;
+extern int softsamples; // this many samples away from the reference sample are averaged. basically filter radius.
 extern const vec3_t vec3_white;
 extern float surflight_subdivide;
 extern int sunsamples;
diff --git a/light/light.cc b/light/light.cc
index 9865fc07..213a4794 100644
--- a/light/light.cc
+++ b/light/light.cc
@@ -1076,7 +1076,7 @@ light_main(int argc, const char **argv)
             softsamples = 1;
             break;
         case 4:
-            softsamples = 2;
+            softsamples = 1;
             break;
         default:
             softsamples = 0;
diff --git a/light/ltface.cc b/light/ltface.cc
index 96870cc0..5af821e1 100644
--- a/light/ltface.cc
+++ b/light/ltface.cc
@@ -880,55 +880,6 @@ Lightmap_Save(lightmapdict_t *lightmaps, const lightsurf_t *lightsurf,
     }
 }
 
-/*
- * Average adjacent points on the grid to soften shadow edges
- */
-static void
-Lightmap_Soften(lightmap_t *lightmap, const lightsurf_t *lightsurf)
-{
-    const int width = (lightsurf->texsize[0] + 1) * oversample;
-    const int height = (lightsurf->texsize[1] + 1) * oversample;
-    const int fullsamples = (2 * softsamples + 1) * (2 * softsamples + 1);
-
-    lightsample_t *softmap = (lightsample_t *) calloc(lightsurf->numpoints, sizeof(lightsample_t));
-    
-    lightsample_t *dst = softmap;
-    for (int i = 0; i < lightsurf->numpoints; i++, dst++) {
-        const int startt = qmax((i / width) - softsamples, 0);
-        const int endt = qmin((i / width) + softsamples + 1, height);
-        const int starts = qmax((i % width) - softsamples, 0);
-        const int ends = qmin((i % width) + softsamples + 1, width);
-
-        for (int t = startt; t < endt; t++) {
-            for (int s = starts; s < ends; s++) {
-                const lightsample_t *src = &lightmap->samples[t * width + s];
-                VectorAdd(dst->color, src->color, dst->color);
-                VectorAdd(dst->direction, src->direction, dst->direction);
-            }
-        }
-        /*
-         * For cases where we are softening near the edge of the lightmap,
-         * take extra samples from the centre point (follows old bjp tools
-         * behaviour)
-         */
-        int samples = (endt - startt) * (ends - starts);
-        if (samples < fullsamples) {
-            const int extraweight = 2 * (fullsamples - samples);
-            const lightsample_t *src = &lightmap->samples[i];
-            VectorMA(dst->color, extraweight, src->color, dst->color);
-            VectorMA(dst->direction, extraweight, src->direction, dst->direction);
-            samples += extraweight;
-        }
-        VectorScale(dst->color, 1.0 / samples, dst->color);
-        VectorScale(dst->direction, 1.0 / samples, dst->direction);
-    }
-
-    memcpy(lightmap->samples, softmap, lightsurf->numpoints * sizeof(lightsample_t));
-    free(softmap);
-}
-
-
-
 /*
  * ============================================================================
  * FACE LIGHTING
@@ -2332,6 +2283,7 @@ WriteLightmaps(const bsp2_t *bsp, bsp2_dface_t *face, facesup_t *facesup, const
     const int actual_height = lightsurf->texsize[1] + 1;
     
     const int oversampled_width = (lightsurf->texsize[0] + 1) * oversample;
+    const int oversampled_height = (lightsurf->texsize[1] + 1) * oversample;
     
     for (int mapnum = 0; mapnum < numstyles; mapnum++) {
         
@@ -2351,11 +2303,16 @@ WriteLightmaps(const bsp2_t *bsp, bsp2_dface_t *face, facesup_t *facesup, const
                 
                 float totalWeight = 0.0f;
                 
-                for (int i = 0; i < oversample; i++) {
-                    for (int j = 0; j < oversample; j++) {
+                for (int i = -softsamples; i < oversample + softsamples; i++) {
+                    for (int j = -softsamples; j < oversample + softsamples; j++) {
                         const int col = (s*oversample) + j;
                         const int row = (t*oversample) + i;
                         
+                        if (col < 0 || col >= oversampled_width)
+                            continue;
+                        if (row < 0 || row >= oversampled_height)
+                            continue;
+                        
                         const int sample_index = (row * oversampled_width) + col;
                         
                         const lightsample_t *sample = &sorted.at(mapnum)->samples[sample_index];
@@ -2582,12 +2539,5 @@ LightFace(bsp2_dface_t *face, facesup_t *facesup, const modelinfo_t *modelinfo,
     /* Apply gamma, rangescale, and clamp */
     LightFace_ScaleAndClamp(lightsurf, lightmaps);
     
-    /* Perform post-processing if requested */
-    if (softsamples > 0) {
-        for (lightmap_t &lightmap : *lightmaps) {
-            Lightmap_Soften(&lightmap, lightsurf);
-        }
-    }
-    
     WriteLightmaps(bsp, face, facesup, lightsurf, lightmaps);
 }