feat: include hidden items in nav tree with hidden flag (#1028)

* feat: emit hidden nav items with `hidden: true` instead of omitting them

Previously, hidden pages/sections/APIs were excluded from the navigation
tree stored in Redis. Now they are included with `hidden: true` so the
docs-site can conditionally show them when the user is browsing a hidden
page. Algolia records are still skipped for hidden items.

Co-Authored-By: Claude <noreply@anthropic.com>

* fix: mark section hidden when all children are hidden

Prevents empty section headers from appearing when consumers filter
hidden items. Also fixes misleading comment ("no visible children"
→ "no children").

Co-Authored-By: Claude <noreply@anthropic.com>

---------

Co-authored-by: dslovinsky <dslovinsky@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>

Author: dslovinsky

src/content-indexer/types/navigation.ts

@@ -2,6 +2,7 @@
 
 interface BaseNavItem {
   title: string;
+  hidden?: boolean;
 }
 
 interface PageNavItem extends BaseNavItem {

src/content-indexer/visitors/__tests__/visit-api-reference.test.ts

@@ -182,7 +182,7 @@ describe("visitApiReference", () => {
     expect(firstPath).toBe("reference/get-balance");
   });
 
-  test("should return no nav for hidden API", () => {
+  test("should mark nav as hidden for hidden API", () => {
     const context = new ProcessingContext("docs");
     const cache = new ContentCache();
 
@@ -215,7 +215,8 @@ describe("visitApiReference", () => {
       navigationAncestors: [],
     });
 
-    expect(result.navItem).toBeUndefined();
+    expect(result.navItem).toBeDefined();
+    expect(result.navItem).toHaveProperty("hidden", true);
     expect(Object.keys(result.indexEntries).length).toBeGreaterThan(0); // Index still created
   });
 

src/content-indexer/visitors/__tests__/visit-page.test.ts

@@ -56,7 +56,7 @@ describe("visitPage", () => {
     });
   });
 
-  test("should skip nav item for hidden page", () => {
+  test("should mark nav item as hidden for hidden page", () => {
     const context = new ProcessingContext("docs");
     const cache = new ContentCache();
 
@@ -74,7 +74,12 @@ describe("visitPage", () => {
       navigationAncestors: [],
     });
 
-    expect(result.navItem).toBeUndefined();
+    expect(result.navItem).toEqual({
+      title: "Hidden Page",
+      path: "/guides/hidden-page",
+      type: "page",
+      hidden: true,
+    });
     expect(result.indexEntries["guides/hidden-page"]).toBeDefined(); // Index still created
   });
 

src/content-indexer/visitors/__tests__/visit-section.test.ts

@@ -181,7 +181,7 @@ describe("visitSection", () => {
     expect(result.indexEntries["guides/page"]).toBeDefined();
   });
 
-  test("should handle hidden section", () => {
+  test("should mark hidden section with hidden flag", () => {
     const context = new ProcessingContext("docs");
     const cache = new ContentCache();
 
@@ -207,11 +207,100 @@ describe("visitSection", () => {
       visitNavigationItem,
     );
 
-    expect(result.navItem).toBeUndefined();
+    expect(result.navItem).toBeDefined();
+    expect(result.navItem).toHaveProperty("hidden", true);
+    expect(result.navItem).toHaveProperty("type", "section");
+    expect(result.navItem).toHaveProperty("title", "Hidden Section");
+    // Children should still be present
+    if (
+      result.navItem &&
+      !Array.isArray(result.navItem) &&
+      "children" in result.navItem
+    ) {
+      expect(result.navItem.children).toHaveLength(1);
+    }
     // Index entries should still be created
     expect(Object.keys(result.indexEntries).length).toBeGreaterThan(0);
   });
 
+  test("should mark section hidden when all children are hidden", () => {
+    const context = new ProcessingContext("docs");
+    const cache = new ContentCache();
+
+    const result = visitSection(
+      {
+        item: {
+          section: "Looks Visible",
+          contents: [
+            {
+              page: "Hidden A",
+              path: "fern/guides/a.mdx",
+              hidden: true,
+            },
+            {
+              page: "Hidden B",
+              path: "fern/guides/b.mdx",
+              hidden: true,
+            },
+          ],
+        },
+        parentPath: PathBuilder.init("guides"),
+        tab: "guides",
+        stripPathPrefix: undefined,
+        contentCache: cache,
+        context,
+        navigationAncestors: [],
+      },
+      visitNavigationItem,
+    );
+
+    // Section itself should be marked hidden since all children are hidden
+    expect(result.navItem).toBeDefined();
+    expect(result.navItem).toHaveProperty("hidden", true);
+    // Children should still be present
+    if (
+      result.navItem &&
+      !Array.isArray(result.navItem) &&
+      "children" in result.navItem
+    ) {
+      expect(result.navItem.children).toHaveLength(2);
+    }
+  });
+
+  test("should not mark section hidden when some children are visible", () => {
+    const context = new ProcessingContext("docs");
+    const cache = new ContentCache();
+
+    const result = visitSection(
+      {
+        item: {
+          section: "Mixed Section",
+          contents: [
+            {
+              page: "Hidden Page",
+              path: "fern/guides/hidden.mdx",
+              hidden: true,
+            },
+            {
+              page: "Visible Page",
+              path: "fern/guides/visible.mdx",
+            },
+          ],
+        },
+        parentPath: PathBuilder.init("guides"),
+        tab: "guides",
+        stripPathPrefix: undefined,
+        contentCache: cache,
+        context,
+        navigationAncestors: [],
+      },
+      visitNavigationItem,
+    );
+
+    expect(result.navItem).toBeDefined();
+    expect(result.navItem).not.toHaveProperty("hidden");
+  });
+
   test("should recursively process nested sections", () => {
     const context = new ProcessingContext("docs");
     const cache = new ContentCache();

src/content-indexer/visitors/processors/__tests__/process-openrpc.test.ts

@@ -158,7 +158,7 @@ describe("processOpenRpcSpec", () => {
     expect(Array.isArray(result.navItem)).toBe(true);
   });
 
-  test("should not create nav if hidden", () => {
+  test("should mark nav as hidden if hidden", () => {
     const context = new ProcessingContext("docs");
 
     const result = processOpenRpcSpec({
@@ -187,7 +187,8 @@ describe("processOpenRpcSpec", () => {
       isFlattened: false,
     });
 
-    expect(result.navItem).toBeUndefined();
+    expect(result.navItem).toBeDefined();
+    expect(result.navItem).toHaveProperty("hidden", true);
     expect(Object.keys(result.indexEntries).length).toBeGreaterThan(0); // Index still created
   });
 

src/content-indexer/visitors/processors/process-openapi.ts

@@ -184,15 +184,11 @@ export const processOpenApiSpec = ({
     tab,
   });
 
-  // Return early if hidden (index only, no navigation)
-  if (isHidden) {
-    return { indexEntries, navItem: undefined };
-  }
-
-  // Create breadcrumb for Algolia
-  const apiSectionBreadcrumb = isFlattened
-    ? undefined
-    : createBreadcrumbNavItem(apiTitle, "api-section");
+  // Create breadcrumb for Algolia (skip for hidden APIs)
+  const apiSectionBreadcrumb =
+    isHidden || isFlattened
+      ? undefined
+      : createBreadcrumbNavItem(apiTitle, "api-section");
 
   // Build navigation items
   const tagSections = buildOpenApiNavigation({
@@ -205,13 +201,14 @@ export const processOpenApiSpec = ({
     isHidden,
   });
 
-  // Return flattened or wrapped navigation
+  // Return flattened or wrapped navigation (marked hidden if applicable)
   const navItem: NavItem | NavItem[] = isFlattened
-    ? tagSections
+    ? tagSections.map((item) => (isHidden ? { ...item, hidden: true } : item))
     : {
         title: apiTitle,
         type: "api-section",
         children: tagSections,
+        ...(isHidden && { hidden: true }),
       };
 
   return { indexEntries, navItem };

src/content-indexer/visitors/processors/process-openrpc.ts

@@ -104,18 +104,16 @@ export const processOpenRpcSpec = ({
     });
   });
 
-  // Return early if hidden
-  if (isHidden) {
-    return { indexEntries, navItem: undefined };
-  }
-
-  // Return flattened or wrapped navigation
+  // Return flattened or wrapped navigation (marked hidden if applicable)
   const navItem: NavItem | NavItem[] = isFlattened
-    ? endpointNavItems
+    ? endpointNavItems.map((item) =>
+        isHidden ? { ...item, hidden: true } : item,
+      )
     : {
         title: apiTitle,
         type: "api-section",
         children: endpointNavItems,
+        ...(isHidden && { hidden: true }),
       };
 
   return { indexEntries, navItem };

src/content-indexer/visitors/visit-page.ts

@@ -54,23 +54,22 @@ export const visitPage = ({
     },
   };
 
-  // Build nav item (skip if hidden)
+  // Build nav item (marked hidden if applicable)
   const descriptionRaw =
     cached?.frontmatter.description || cached?.frontmatter.subtitle;
   const description =
     typeof descriptionRaw === "string" ? descriptionRaw : undefined;
 
-  const navItem: NavItem | undefined = pageItem.hidden
-    ? undefined
-    : {
-        title: pageItem.page,
-        path: `/${finalPath}`,
-        type: "page",
-        description,
-      };
+  const navItem: NavItem = {
+    title: pageItem.page,
+    path: `/${finalPath}`,
+    type: "page",
+    description,
+    ...(pageItem.hidden && { hidden: true }),
+  };
 
   // Build Algolia record (if content available and not hidden)
-  if (cached && navItem) {
+  if (cached && !pageItem.hidden) {
     const title = cached.frontmatter.title || pageItem.page;
     context.addAlgoliaRecord({
       pageType: "Guide",

src/content-indexer/visitors/visit-section.ts

@@ -138,13 +138,19 @@ export const visitSection = (
     .flat()
     .filter((child): child is NavItem => child !== undefined);
 
-  // Only include section in nav if it has children and is not hidden
-  if (children.length === 0 || sectionItem.hidden) {
+  // Skip section from nav if it has no children
+  if (children.length === 0) {
     return { indexEntries, navItem: undefined };
   }
 
-  // Update section nav item with children
+  // Update section nav item with children.
+  // Mark hidden if explicitly hidden OR if every child is hidden
+  // (prevents empty section headers when consumers filter hidden items).
   sectionNavItem.children = children;
+  const allChildrenHidden = children.every((child) => child.hidden === true);
+  if (sectionItem.hidden || allChildrenHidden) {
+    sectionNavItem.hidden = true;
+  }
 
   return { indexEntries, navItem: sectionNavItem };
 };