light: make sky faces not occlude lights

include/light/trace.hh

@@ -57,9 +57,14 @@ hittype_t DirtTrace(const vec3_t start, const vec3_t dirn, vec_t dist, const dmo
 // used for CalcPoints
 bool IntersectSingleModel(const vec3_t start, const vec3_t dir, vec_t dist, const dmodel_t *self, vec_t *hitdist_out);
 
+enum class tracetype_t {
+    NORMAL,
+    TEST_SKY
+};
+
 class raystream_t {
 public:
-    virtual void pushRay(int i, const vec_t *origin, const vec3_t dir, float dist, const dmodel_t *selfshadow, const vec_t *color = nullptr, const vec_t *normalcontrib = nullptr) = 0;
+    virtual void pushRay(int i, const vec_t *origin, const vec3_t dir, float dist, const dmodel_t *selfshadow, tracetype_t type, const vec_t *color = nullptr, const vec_t *normalcontrib = nullptr) = 0;
     virtual size_t numPushedRays() = 0;
     virtual void tracePushedRaysOcclusion() = 0;
     virtual void tracePushedRaysIntersection() = 0;

light/entities.cc

@@ -1157,7 +1157,7 @@ void EstimateVisibleBoundsAtPoint(const vec3_t point, vec3_t mins, vec3_t maxs)
             vec3_t dir;
             UniformPointOnSphere(dir, u1, u2);
         
-            rs->pushRay(0, point, dir, 65536.0f, nullptr);
+            rs->pushRay(0, point, dir, 65536.0f, nullptr, tracetype_t::NORMAL);
         }
     }
     

light/ltface.cc

@@ -1354,7 +1354,7 @@ LightFace_Entity(const bsp2_t *bsp,
             continue;
         }
         
-        rs->pushRay(i, surfpoint, surfpointToLightDir, surfpointToLightDist, shadowself, color, normalcontrib);
+        rs->pushRay(i, surfpoint, surfpointToLightDir, surfpointToLightDist, shadowself, tracetype_t::NORMAL, color, normalcontrib);
     }
     
     rs->tracePushedRaysOcclusion();
@@ -1432,7 +1432,7 @@ LightFace_Sky(const sun_t *sun, const lightsurf_t *lightsurf, lightmapdict_t *li
             continue;
         }
         
-        rs->pushRay(i, surfpoint, incoming, MAX_SKY_DIST, shadowself);
+        rs->pushRay(i, surfpoint, incoming, MAX_SKY_DIST, shadowself, tracetype_t::TEST_SKY);
     }
     
     rs->tracePushedRaysIntersection();
@@ -1540,7 +1540,7 @@ LightFace_Min(const bsp2_t *bsp, const bsp2_dface_t *face,
                 vec3_t surfpointToLightDir;
                 vec_t surfpointToLightDist = GetDir(surfpoint, *entity.origin.vec3Value(), surfpointToLightDir);
                 
-                rs->pushRay(i, surfpoint, surfpointToLightDir, surfpointToLightDist, shadowself);
+                rs->pushRay(i, surfpoint, surfpointToLightDir, surfpointToLightDist, shadowself, tracetype_t::NORMAL);
             }
         }
         
@@ -1760,7 +1760,7 @@ LightFace_Bounce(const bsp2_t *bsp, const bsp2_dface_t *face, const lightsurf_t
             if (LightSample_Brightness(indirect) < 0.25)
                 continue;
             
-            rs->pushRay(i, vpl.pos, dir, dist, /*shadowself*/ nullptr, indirect);
+            rs->pushRay(i, vpl.pos, dir, dist, /*shadowself*/ nullptr, tracetype_t::NORMAL, indirect);
         }
         
         total_bounce_rays += rs->numPushedRays();
@@ -1968,7 +1968,7 @@ DirtAtPoint(const globalconfig_t &cfg, raystream_t *rs, const vec3_t point, cons
         vec3_t dir;
         TransformToTangentSpace(normal, myUp, myRt, dirtvec, dir);
         
-        rs->pushRay(j, point, dir, cfg.dirtDepth.floatValue(), selfshadow);
+        rs->pushRay(j, point, dir, cfg.dirtDepth.floatValue(), selfshadow, tracetype_t::NORMAL);
     }
     
     Q_assert(rs->numPushedRays() == numDirtVectors);
@@ -2035,7 +2035,7 @@ LightFace_CalculateDirt(lightsurf_t *lightsurf)
             vec3_t dir;
             TransformToTangentSpace(lightsurf->normals[i], myUps[i], myRts[i], dirtvec, dir);
             
-            rs->pushRay(i, lightsurf->points[i], dir, cfg.dirtDepth.floatValue(), selfshadow);
+            rs->pushRay(i, lightsurf->points[i], dir, cfg.dirtDepth.floatValue(), selfshadow, tracetype_t::NORMAL);
         }
         
         Q_assert(rs->numPushedRays() == lightsurf->numpoints);

light/trace.cc

@@ -875,7 +875,8 @@ public:
     
     raystream_bsp_t() {}
     
-    virtual void pushRay(int i, const vec_t *origin, const vec3_t dir, float dist, const dmodel_t *selfshadow, const vec_t *color = nullptr, const vec_t *normalcontrib = nullptr) {
+    virtual void pushRay(int i, const vec_t *origin, const vec3_t dir, float dist, const dmodel_t *selfshadow, tracetype_t type, const vec_t *color = nullptr, const vec_t *normalcontrib = nullptr) {
+        // FIXME: `type` not implemeted (sky test)
         bsp_ray_t r { i, origin, dir, dist, selfshadow, color, normalcontrib };
         _rays.push_back(r);
         Q_assert(_rays.size() <= _maxrays);

light/trace_embree.cc

@@ -151,7 +151,7 @@ CreateGeometryFromWindings(RTCScene scene, const std::vector<winding_t *> &windi
 
 // Creates a scene with just the faces in this model,
 // used by CalcPoints.
-// Liquids are left out but sky faces are included
+// Liquids are left out, as are sky faces
 RTCScene CreatePerModelScene(RTCDevice device, const bsp2_t *bsp, const dmodel_t *model) {
     std::vector<const bsp2_dface_t *> faces;
     
@@ -159,9 +159,9 @@ RTCScene CreatePerModelScene(RTCDevice device, const bsp2_t *bsp, const dmodel_t
         const bsp2_dface_t *face = &bsp->dfaces[model->firstface + i];
         
         const char *texname = Face_TextureName(bsp, face);
-        if (texname[0] == '*') {
+        const int contents = TextureName_Contents(texname);
-            // ignore liquids
+        
-        } else {
+        if (contents == CONTENTS_SOLID) {
             faces.push_back(face);
         }
     }
@@ -231,6 +231,16 @@ enum class filtertype_t {
     INTERSECTION, OCCLUSION
 };
 
+/**
+ * Set this bit if the ray is coming from a selfshadow model
+ */
+static const unsigned RAYMASK_FROM_SELFSHADOWGEOM = 1;
+/**
+ * Set this bit if the ray should hit sky faces. This only happens when we
+ * are testing sunlight; normal lights pass through sky faces
+ */
+static const unsigned RAYMASK_TEST_SKY_FACES = 2;
+
 // called to evaluate transparency
 template<filtertype_t filtertype>
 static void
@@ -254,27 +264,37 @@ Embree_FilterFuncN(int* valid,
         const unsigned &geomID = RTCHitN_geomID(potentialHit, N, i);
         const unsigned &primID = RTCHitN_primID(potentialHit, N, i);
         
-        // bail if we hit a selfshadow face, but the ray is not coming from within that model
+        if (geomID == skygeom.geomID) {
-        if (mask == 0 && geomID == selfshadowgeom.geomID) {
+            if (!(mask & RAYMASK_TEST_SKY_FACES)) {
-            // reject hit
-            valid[i] = INVALID;
-            continue;
-        }
-        
-        // test fence texture
-        const bsp2_dface_t *face = Embree_LookupFace(geomID, primID);
-        
-        const char *name = Face_TextureName(bsp_static, face);
-        if (name[0] == '{') {
-            vec3_t hitpoint;
-            Embree_RayEndpoint(ray, potentialHit, N, i, hitpoint);
-            const int sample = SampleTexture(face, bsp_static, hitpoint);
-            
-            if (sample == 255) {
                 // reject hit
                 valid[i] = INVALID;
                 continue;
             }
+        } else if (geomID == selfshadowgeom.geomID) {
+            // bail if we hit a selfshadow face, but the ray is not coming from within that model
+            if (!(mask & RAYMASK_FROM_SELFSHADOWGEOM)) {
+                // reject hit
+                valid[i] = INVALID;
+                continue;
+            }
+        } else if (geomID == fencegeom.geomID) {
+            // test fence texture
+            const bsp2_dface_t *face = Embree_LookupFace(geomID, primID);
+            
+            const char *name = Face_TextureName(bsp_static, face);
+            if (name[0] == '{') {
+                vec3_t hitpoint;
+                Embree_RayEndpoint(ray, potentialHit, N, i, hitpoint);
+                const int sample = SampleTexture(face, bsp_static, hitpoint);
+                
+                if (sample == 255) {
+                    // reject hit
+                    valid[i] = INVALID;
+                    continue;
+                }
+            }
+        } else {
+            Q_assert_unreachable();
         }
         
         // accept hit
@@ -543,12 +563,15 @@ Embree_TraceInit(const bsp2_t *bsp)
     selfshadowgeom = CreateGeometry(bsp, scene, selfshadowfaces);
     CreateGeometryFromWindings(scene, skipwindings);
     
+    rtcSetIntersectionFilterFunctionN(scene, skygeom.geomID, Embree_FilterFuncN<filtertype_t::INTERSECTION>);
+    rtcSetOcclusionFilterFunctionN(scene, skygeom.geomID, Embree_FilterFuncN<filtertype_t::OCCLUSION>);
+    
     rtcSetIntersectionFilterFunctionN(scene, fencegeom.geomID, Embree_FilterFuncN<filtertype_t::INTERSECTION>);
     rtcSetOcclusionFilterFunctionN(scene, fencegeom.geomID, Embree_FilterFuncN<filtertype_t::OCCLUSION>);
     
     rtcSetIntersectionFilterFunctionN(scene, selfshadowgeom.geomID, Embree_FilterFuncN<filtertype_t::INTERSECTION>);
     rtcSetOcclusionFilterFunctionN(scene, selfshadowgeom.geomID, Embree_FilterFuncN<filtertype_t::OCCLUSION>);
-
+    
     rtcCommit (scene);
     
     logprint("Embree_TraceInit: %d skyfaces %d solidfaces %d fencefaces %d selfshadowfaces %d skipwindings\n",
@@ -573,8 +596,14 @@ static RTCRay SetupRay(const vec3_t start, const vec3_t dir, vec_t dist, const d
     ray.instID = RTC_INVALID_GEOMETRY_ID;
     
     // NOTE: we are not using the ray masking feature of embree, but just using
-    // this field to store whether the ray is coming from self-shadow geometry
+    // this field to store whether the ray is coming from self-shadow geometry.
-    ray.mask = (self == nullptr) ? 0 : 1;
+    // and also whether we want to hit sky faces.
+    unsigned mask = 0;
+    if (self != nullptr) {
+        mask |= RAYMASK_FROM_SELFSHADOWGEOM;
+    }
+    
+    ray.mask = mask;
     ray.time = 0.f;
     return ray;
 }
@@ -612,6 +641,7 @@ qboolean Embree_TestSky(const vec3_t start, const vec3_t dirn, const dmodel_t *s
     VectorNormalize(dir_normalized);
     
     RTCRay ray = SetupRay(start, dir_normalized, MAX_SKY_RAY_DEPTH, self);
+    ray.mask |= RAYMASK_TEST_SKY_FACES;
     rtcIntersect(scene, ray);
 
     qboolean hit_sky = (ray.geomID == skygeom.geomID);
@@ -728,9 +758,14 @@ public:
         free(_ray_normalcontribs);
     }
     
-    virtual void pushRay(int i, const vec_t *origin, const vec3_t dir, float dist, const dmodel_t *selfshadow, const vec_t *color = nullptr, const vec_t *normalcontrib = nullptr) {
+    virtual void pushRay(int i, const vec_t *origin, const vec3_t dir, float dist, const dmodel_t *selfshadow, tracetype_t type, const vec_t *color = nullptr, const vec_t *normalcontrib = nullptr) {
         Q_assert(_numrays<_maxrays);
         _rays[_numrays] = SetupRay(origin, dir, dist, selfshadow);
+        
+        if (type == tracetype_t::TEST_SKY) {
+            _rays[_numrays].mask |= RAYMASK_TEST_SKY_FACES;
+        }
+        
         _rays_maxdist[_numrays] = dist;
         _point_indices[_numrays] = i;
         if (color) {