qbsp: experimental -filldetail feature (defaults to on)

also change default -filltype from auto to inside

include/qbsp/outside.hh

@@ -35,3 +35,5 @@ bool FillOutside(tree_t &tree, hull_index_t hullnum, bspbrush_t::container &brus
 void MarkBrushSidesInvisible(bspbrush_t::container &brushes);
 
 void FillBrushEntity(tree_t &tree, hull_index_t hullnum, bspbrush_t::container &brushes);
+
+void FillDetail(tree_t &tree, hull_index_t hullnum, bspbrush_t::container &brushes);

include/qbsp/qbsp.hh

@@ -224,6 +224,7 @@ public:
     setting_set texturedefs;
     setting_numeric<vec_t> lmscale;
     setting_enum<filltype_t> filltype;
+    setting_bool filldetail;
     setting_invertible_bool allow_upgrade;
     setting_validator<setting_int32> maxedges;
     setting_numeric<vec_t> midsplitbrushfraction;

qbsp/outside.cc

@@ -43,6 +43,17 @@ static bool LeafSealsMap(const node_t *node)
     return qbsp_options.target_game->contents_seals_map(node->contents);
 }
 
+static bool LeafSealsForDetailFill(const node_t *node)
+{
+    Q_assert(node->is_leaf);
+
+    // NOTE: detail-solid is considered sealing for the detail fill,
+    // but not the regular fill (LeafSealsMap).
+
+    return qbsp_options.target_game->contents_are_any_solid(node->contents) ||
+           qbsp_options.target_game->contents_are_sky(node->contents);
+}
+
 /*
 ===========
 PointInLeaf
@@ -109,6 +120,16 @@ static bool OutsideFill_Passable(const portal_t *p)
     return !LeafSealsMap(p->nodes[0]) && !LeafSealsMap(p->nodes[1]);
 }
 
+static bool DetailFill_Passable(const portal_t *p)
+{
+    if (!p->onnode) {
+        // portal to outside_node
+        return false;
+    }
+
+    return !LeafSealsForDetailFill(p->nodes[0]) && !LeafSealsForDetailFill(p->nodes[1]);
+}
+
 /*
 ==================
 FloodFillLeafsFromVoid
@@ -406,7 +427,7 @@ static void MarkVisibleBrushSides_R(node_t *node)
         return;
     }
 
-    if (LeafSealsMap(node)) {
+    if (LeafSealsForDetailFill(node)) {
         // this leaf is opaque
         return;
     }
@@ -478,6 +499,35 @@ static void OutLeafsToSolid_R(node_t *node, settings::filltype_t filltype, outle
     stats.outleafs++;
 }
 
+struct detail_filled_leafs_stats_t : logging::stat_tracker_t
+{
+    stat &filledleafs = register_stat("detail filled leafs", true);
+};
+
+static void FillDetailEnclosedLeafsToDetailSolid_R(node_t *node, detail_filled_leafs_stats_t &stats)
+{
+    if (!node->is_leaf) {
+        FillDetailEnclosedLeafsToDetailSolid_R(node->children[0], stats);
+        FillDetailEnclosedLeafsToDetailSolid_R(node->children[1], stats);
+        return;
+    }
+
+    // skip leafs reachable from entities
+    if (node->occupied > 0) {
+        return;
+    }
+
+    // Don't fill sky, or count solids as outleafs
+    if (LeafSealsForDetailFill(node)) {
+        return;
+    }
+
+    // Finally, we can fill it in as detail solid.
+    node->contents =
+        qbsp_options.target_game->create_detail_solid_contents(qbsp_options.target_game->create_solid_contents());
+    stats.filledleafs++;
+}
+
 //=============================================================================
 
 #if 0
@@ -492,12 +542,17 @@ static void SetOccupied_R(node_t *node, int dist)
 }
 #endif
 
+using portal_passable_t = bool (*)(const portal_t *);
+
 /*
 ==================
 precondition: all leafs have occupied set to 0
+
+sets node->occupied to 1 or more to indicate the number of steps to a directly occupied leaf
 ==================
 */
-static void BFSFloodFillFromOccupiedLeafs(const std::vector<node_t *> &occupied_leafs)
+static void BFSFloodFillFromOccupiedLeafs(
+    const std::vector<node_t *> &occupied_leafs, const portal_passable_t &predicate)
 {
     std::list<std::pair<node_t *, int>> queue;
     for (node_t *leaf : occupied_leafs) {
@@ -521,7 +576,7 @@ static void BFSFloodFillFromOccupiedLeafs(const std::vector<node_t *> &occupied_
             for (portal_t *portal = node->portals; portal; portal = portal->next[!side]) {
                 side = (portal->nodes[0] == node);
 
-                if (!OutsideFill_Passable(portal))
+                if (!predicate(portal))
                     continue;
 
                 node_t *neighbour = portal->nodes[side];
@@ -646,7 +701,7 @@ bool FillOutside(tree_t &tree, hull_index_t hullnum, bspbrush_t::container &brus
     }
 
     if (filltype == settings::filltype_t::INSIDE) {
-        BFSFloodFillFromOccupiedLeafs(occupied_leafs);
+        BFSFloodFillFromOccupiedLeafs(occupied_leafs, OutsideFill_Passable);
 
         /* first check to see if an occupied leaf is hit */
         const int side = (tree.outside_node.portals->nodes[0] == &tree.outside_node);
@@ -753,3 +808,38 @@ void FillBrushEntity(tree_t &tree, hull_index_t hullnum, bspbrush_t::container &
 
     MarkVisibleBrushSides_R(tree.headnode);
 }
+
+/**
+ * Searches for empty pockets that are fully enclosed by solid or detail|solid and not reachable by entities
+ *
+ * Intended to be run after FillOutside, so we preserve the visibility flag on brush sides, but
+ * additionally mark some new brush sides as invisible.
+ */
+void FillDetail(tree_t &tree, hull_index_t hullnum, bspbrush_t::container &brushes)
+{
+    logging::funcheader();
+
+    // Clear the outside filling state on all leafs
+    ClearOccupied_r(tree.headnode);
+
+    // Sets leaf->occupant
+    MarkOccupiedLeafs(tree.headnode, hullnum);
+    const std::vector<node_t *> occupied_leafs = FindOccupiedLeafs(tree.headnode);
+
+    if (occupied_leafs.empty()) {
+        logging::print("WARNING: No entities in empty space -- no filling performed (hull {})\n", hullnum.value_or(0));
+        return;
+    }
+
+    BFSFloodFillFromOccupiedLeafs(occupied_leafs, DetailFill_Passable);
+
+    // change the leaf contents
+    detail_filled_leafs_stats_t stats;
+    FillDetailEnclosedLeafsToDetailSolid_R(tree.headnode, stats);
+
+    // See missing_face_simple.map for a test case with a brush that straddles between void and non-void
+
+    MarkBrushSidesInvisible(brushes);
+
+    MarkVisibleBrushSides_R(tree.headnode);
+}

qbsp/qbsp.cc

@@ -542,10 +542,12 @@ qbsp_settings::qbsp_settings()
           "path to a texture definition file, which can transform textures in the .map into other textures."},
       lmscale{this, "lmscale", 1.0, &common_format_group,
           "change global lmscale (force _lmscale key on all entities). outputs the LMSCALE BSPX lump."},
-      filltype{this, "filltype", filltype_t::AUTO,
+      filltype{this, "filltype", filltype_t::INSIDE,
           {{"auto", filltype_t::AUTO}, {"inside", filltype_t::INSIDE}, {"outside", filltype_t::OUTSIDE}},
           &common_format_group,
           "whether to fill the map from the outside in (lenient), from the inside out (aggressive), or to automatically decide based on the hull being used."},
+      filldetail{this, "filldetail", true, &common_format_group,
+          "whether to fill in empty spaces which are fully enclosed by detail solid"},
       allow_upgrade{this, "allowupgrade", true, &common_format_group,
           "allow formats to \"upgrade\" to compatible extended formats when a limit is exceeded (ie Quake BSP to BSP2)"},
       maxedges{[](setting_int32 &setting) { return setting.value() == 0 || setting.value() >= 3; }, this, "maxedges",
@@ -1052,6 +1054,9 @@ static void ProcessEntity(mapentity_t &entity, hull_index_t hullnum)
             // assume non-world bmodels are simple
             MakeTreePortals(tree);
             if (FillOutside(tree, hullnum, brushes)) {
+                if (qbsp_options.filldetail.value())
+                    FillDetail(tree, hullnum, brushes);
+
                 // make a really good tree
                 tree.clear();
                 BrushBSP(tree, entity, brushes, tree_split_t::PRECISE);
@@ -1059,6 +1064,9 @@ static void ProcessEntity(mapentity_t &entity, hull_index_t hullnum)
                 // fill again so PruneNodes works
                 MakeTreePortals(tree);
                 FillOutside(tree, hullnum, brushes);
+                if (qbsp_options.filldetail.value())
+                    FillDetail(tree, hullnum, brushes);
+
                 FreeTreePortals(tree);
                 PruneNodes(tree.headnode);
             }
@@ -1101,6 +1109,9 @@ static void ProcessEntity(mapentity_t &entity, hull_index_t hullnum)
         // we can skip using them as BSP splitters on the "really good tree"
         // (effectively expanding those brush sides outwards).
         if (!qbsp_options.nofill.value() && FillOutside(tree, hullnum, brushes)) {
+            if (qbsp_options.filldetail.value())
+                FillDetail(tree, hullnum, brushes);
+
             // make a really good tree
             tree.clear();
             BrushBSP(tree, entity, brushes, tree_split_t::PRECISE);
@@ -1124,6 +1135,9 @@ static void ProcessEntity(mapentity_t &entity, hull_index_t hullnum)
 
             // fill again so PruneNodes works
             FillOutside(tree, hullnum, brushes);
+
+            if (qbsp_options.filldetail.value())
+                FillDetail(tree, hullnum, brushes);
         }
 
         // Area portals