From 9fa9124404967fec40ec12f90ddec74e601dd0d2 Mon Sep 17 00:00:00 2001
From: Eric Wasylishen <ewasylishen@gmail.com>
Date: Wed, 30 Nov 2022 23:02:40 -0700
Subject: [PATCH] light: redesign Q2 phong check

- always do angle check
- always do gamedef specific exclusions (e.g. warping surfflags don't phong with non-warping)
- if Q2 style phong is requested, but Q1 is not in use, use the default phong angle
- always do smoothing group ID check (preparation for supporting this in Q1 mode via a key)
---
 include/light/light.hh |  1 +
 light/phong.cc         | 37 ++++++++++++++++---------------------
 2 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/include/light/light.hh b/include/light/light.hh
index 92800fbe..b313699a 100644
--- a/include/light/light.hh
+++ b/include/light/light.hh
@@ -193,6 +193,7 @@ extern int dump_vertnum;
 
 class modelinfo_t : public settings::setting_container
 {
+public:
     static constexpr vec_t DEFAULT_PHONG_ANGLE = 89.0;
 
 public:
diff --git a/light/phong.cc b/light/phong.cc
index dbb0b7c4..865748dc 100644
--- a/light/phong.cc
+++ b/light/phong.cc
@@ -452,25 +452,6 @@ void CalculateVertexNormals(const mbsp_t *bsp)
     for (auto &f : bsp->dfaces) {
         // Q2 shading groups
         const int f_phongValue = Q2_FacePhongValue(bsp, &f);
-        if (f_phongValue) {
-            for (int j = 0; j < f.numedges; j++) {
-                const int v = Face_VertexAtIndex(bsp, &f, j);
-                // walk over all faces incident to f (we will walk over neighbours multiple times, doesn't matter)
-                for (const mface_t *f2 : vertsToFaces[v]) {
-                    if (f2 == &f)
-                        continue;
-
-                    const int f2_phongValue = Q2_FacePhongValue(bsp, f2);
-                    if (f_phongValue != f2_phongValue)
-                        continue;
-
-                    // we've already checked f_phongValue is nonzero, so smooth these two faces.
-                    smoothFaces[&f].insert(f2);
-                }
-            }
-
-            continue;
-        }
 
         // Q1 phong angle stuff
         auto *f_texinfo = Face_Texinfo(bsp, &f);
@@ -479,7 +460,11 @@ void CalculateVertexNormals(const mbsp_t *bsp)
         const qplane3d f_plane = Face_Plane(bsp, &f);
 
         // any face normal within this many degrees can be smoothed with this face
-        const vec_t &f_phong_angle = extended_texinfo_flags[f.texinfo].phong_angle;
+        vec_t f_phong_angle = extended_texinfo_flags[f.texinfo].phong_angle;
+        if (f_phong_angle == 0 && f_phongValue != 0) {
+            // if Q2 style phong is requested, but Q1 is not in use, set the default phong angle
+            f_phong_angle = modelinfo_t::DEFAULT_PHONG_ANGLE;
+        }
         vec_t f_phong_angle_concave = extended_texinfo_flags[f.texinfo].phong_angle_concave;
         if (f_phong_angle_concave == 0) {
             f_phong_angle_concave = f_phong_angle;
@@ -497,7 +482,12 @@ void CalculateVertexNormals(const mbsp_t *bsp)
                     continue;
 
                 // FIXME: factor out and share with above?
-                const vec_t &f2_phong_angle = extended_texinfo_flags[f2->texinfo].phong_angle;
+                const int f2_phongValue = Q2_FacePhongValue(bsp, f2);
+                vec_t f2_phong_angle = extended_texinfo_flags[f2->texinfo].phong_angle;
+                if (f2_phong_angle == 0 && f2_phongValue != 0) {
+                    // if Q2 style phong is requested, but Q1 is not in use, set the default phong angle
+                    f2_phong_angle = modelinfo_t::DEFAULT_PHONG_ANGLE;
+                }
                 vec_t f2_phong_angle_concave = extended_texinfo_flags[f2->texinfo].phong_angle_concave;
                 if (f2_phong_angle_concave == 0) {
                     f2_phong_angle_concave = f2_phong_angle;
@@ -527,6 +517,11 @@ void CalculateVertexNormals(const mbsp_t *bsp)
                 const vec_t min_threshold = min(f_threshold, f2_threshold);
                 const vec_t cosmaxangle = cos(DEG2RAD(min_threshold));
 
+                if (f_phongValue != f2_phongValue) {
+                    // mismatched smoothing groups never phong
+                    continue;
+                }
+
                 // check the angle between the face normals
                 if (cosangle >= cosmaxangle) {
                     smoothFaces[&f].insert(f2);