diff --git a/common/mathlib.cc b/common/mathlib.cc index 77dd6b45..1b5e203f 100644 --- a/common/mathlib.cc +++ b/common/mathlib.cc @@ -357,6 +357,18 @@ GLM_MakeInwardFacingEdgePlanes(std::vector points) result.push_back(edgeplane.second); } + // For each edge plane, check that every point in `points` is on or above it + // If this fails, the polygon is not convex. + for (const vec4 &edgeplane : result) { + for (const vec3 &point : points) { + const float distAbove = GLM_DistAbovePlane(edgeplane, point); + if (distAbove < -POINT_EQUAL_EPSILON) { + // Non-convex polygon + return {}; + } + } + } + return result; } diff --git a/light/test_light.cc b/light/test_light.cc index dc6fa696..09a39fae 100644 --- a/light/test_light.cc +++ b/light/test_light.cc @@ -156,6 +156,19 @@ static void checkBox(const vector &edges, const vector &poly) { } +TEST(mathlib, EdgePlanesOfNonConvexPoly) { + // hourglass, non-convex + const vector poly { + { 0,0,0 }, + { 64,64,0 }, + { 0,64,0 }, + { 64,0,0 } + }; + + const auto edges = GLM_MakeInwardFacingEdgePlanes(poly); + EXPECT_EQ(vector(), edges); +} + TEST(mathlib, PointInPolygon) { // clockwise const vector poly {