From 412e169ef6745f0aac3ca78091660664cc2d94fe Mon Sep 17 00:00:00 2001
From: Eric Wasylishen <ewasylishen@gmail.com>
Date: Fri, 10 Jun 2016 12:23:22 -0600
Subject: [PATCH] light: add cull_backfaces hack used by CalcPoints, to handle
 the case when a shadow-casting bmodel is obstructing the midpoint of a face.
 probably should be removed and CalcPoints made more robust.

---
 include/light/light.h |  2 +-
 light/ltface.c        |  8 ++++----
 light/trace.c         | 13 +++++++++++--
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/include/light/light.h b/include/light/light.h
index c5f8b585..97b528f1 100644
--- a/include/light/light.h
+++ b/include/light/light.h
@@ -42,7 +42,7 @@ extern "C" {
  */
 qboolean TestSky(const vec3_t start, const vec3_t dirn, const dmodel_t *self);
 qboolean TestLight(const vec3_t start, const vec3_t stop, const dmodel_t *self);
-qboolean DirtTrace(const vec3_t start, const vec3_t stop, const dmodel_t *self, vec3_t hitpoint_out, plane_t *hitplane_out, const bsp2_dface_t **face_out);
+qboolean DirtTrace(const vec3_t start, const vec3_t stop, const dmodel_t *self, bool cull_backfaces, vec3_t hitpoint_out, plane_t *hitplane_out, const bsp2_dface_t **face_out);
 
 int
 SampleTexture(const bsp2_dface_t *face, const bsp2_t *bsp, const vec3_t point);
diff --git a/light/ltface.c b/light/ltface.c
index 8a9ae26e..90fc0082 100644
--- a/light/ltface.c
+++ b/light/ltface.c
@@ -604,7 +604,7 @@ CheckObstructed(const lightsurf_t *surf, const vec3_t offset, const vec_t us, co
             
             vec3_t hitpoint = {0};
             plane_t hitplane = {0};
-            if (DirtTrace(surf->midpoint, testpoint, surf->modelinfo->model, hitpoint, &hitplane, NULL)) {
+            if (DirtTrace(surf->midpoint, testpoint, surf->modelinfo->model, true, hitpoint, &hitplane, NULL)) {
                 // make a corrected point
                 
                 vec3_t tracedir;
@@ -1869,7 +1869,7 @@ DirtForSample(const dmodel_t *model, const vec3_t origin, const vec3_t normal){
             VectorMA( origin, dirtDepth.value, direction, traceEnd );
 
             /* trace */
-            if (DirtTrace(origin, traceEnd, model, traceHitpoint, NULL, NULL)) {
+            if (DirtTrace(origin, traceEnd, model, false, traceHitpoint, NULL, NULL)) {
                 VectorSubtract( traceHitpoint, origin, displacement );
                 gatherDirt += 1.0f - ooDepth * VectorLength( displacement );
             }
@@ -1886,7 +1886,7 @@ DirtForSample(const dmodel_t *model, const vec3_t origin, const vec3_t normal){
             VectorMA( origin, dirtDepth.value, direction, traceEnd );
             
             /* trace */
-            if (DirtTrace(origin, traceEnd, model, traceHitpoint, NULL, NULL)) {
+            if (DirtTrace(origin, traceEnd, model, false, traceHitpoint, NULL, NULL)) {
                 VectorSubtract( traceHitpoint, origin, displacement );
                 gatherDirt += 1.0f - ooDepth * VectorLength( displacement );
             }
@@ -1897,7 +1897,7 @@ DirtForSample(const dmodel_t *model, const vec3_t origin, const vec3_t normal){
     VectorMA( origin, dirtDepth.value, normal, traceEnd );
     
     /* trace */
-    if (DirtTrace(origin, traceEnd, model, traceHitpoint, NULL, NULL)) {
+    if (DirtTrace(origin, traceEnd, model, false, traceHitpoint, NULL, NULL)) {
         VectorSubtract( traceHitpoint, origin, displacement );
         gatherDirt += 1.0f - ooDepth * VectorLength( displacement );
     }
diff --git a/light/trace.c b/light/trace.c
index fc7f09e7..d1a0dbcf 100644
--- a/light/trace.c
+++ b/light/trace.c
@@ -30,6 +30,8 @@ typedef struct traceinfo_s {
     vec3_t			point;
     const bsp2_dface_t          *face;
     plane_t  hitplane;
+    bool option_cull_backface;
+    
     /* returns true if sky was hit. */
     bool hitsky;
     bool hitback;
@@ -653,13 +655,18 @@ TestSky(const vec3_t start, const vec3_t dirn, const dmodel_t *self)
  *
  * returns true if the trace from start to stop hits something solid,
  * or if it started in the void.
+ *
+ * cull_backfaces is a hack used by CalcPoints, to handle the case when a
+ * shadow-casting bmodel is obstructing the midpoint of a face.
+ * probably should be removed and CalcPoints made more robust.
  * ============
  */
 qboolean
-DirtTrace(const vec3_t start, const vec3_t stop, const dmodel_t *self, vec3_t hitpoint_out, plane_t *hitplane_out, const bsp2_dface_t **face_out)
+DirtTrace(const vec3_t start, const vec3_t stop, const dmodel_t *self, bool cull_backfaces, vec3_t hitpoint_out, plane_t *hitplane_out, const bsp2_dface_t **face_out)
 {
     const modelinfo_t *const *model;
     traceinfo_t ti = {0};
+    ti.option_cull_backface = cull_backfaces;
     
     VectorSubtract(stop, start, ti.dir);
     VectorNormalize(ti.dir);
@@ -769,7 +776,9 @@ bool TraceFaces (traceinfo_t *ti, int node, const vec3_t start, const vec3_t end
                 // check if we hit the back side
                 ti->hitback = (DotProduct(ti->dir, fi->plane.normal) >= 0);
                 
-                return true;
+                if (!(ti->option_cull_backface && ti->hitback)) {
+                    return true;
+                }
             }
         }