common: GLM_MakeInwardFacingEdgePlanes: handle non-convex poly

common/mathlib.cc

@@ -357,6 +357,18 @@ GLM_MakeInwardFacingEdgePlanes(std::vector<vec3> points)
         result.push_back(edgeplane.second);
     }
     
+    // For each edge plane, check that every point in `points` is on or above it
+    // If this fails, the polygon is not convex.
+    for (const vec4 &edgeplane : result) {
+        for (const vec3 &point : points) {
+            const float distAbove = GLM_DistAbovePlane(edgeplane, point);
+            if (distAbove < -POINT_EQUAL_EPSILON) {
+                // Non-convex polygon
+                return {};
+            }
+        }
+    }
+    
     return result;
 }
 

light/test_light.cc

@@ -156,6 +156,19 @@ static void checkBox(const vector<vec4> &edges, const vector<vec3> &poly) {
 
 }
 
+TEST(mathlib, EdgePlanesOfNonConvexPoly) {
+    // hourglass, non-convex
+    const vector<vec3> poly {
+        { 0,0,0 },
+        { 64,64,0 },
+        { 0,64,0 },
+        { 64,0,0 }
+    };
+    
+    const auto edges = GLM_MakeInwardFacingEdgePlanes(poly);
+    EXPECT_EQ(vector<vec4>(), edges);
+}
+
 TEST(mathlib, PointInPolygon) {
     // clockwise
     const vector<vec3> poly {