nickcernera/sitemap-cleanup (#411)

* chore: add proper sitemap generation, add lastmod to sitemap, prepare for algolia crawl

* chore: add generate-pages script to yarn dev

* test: testing script to detect file moves via git

* enhancement: auto-detect reversions of redirects

* chore: cleaning deprecated redirects

* chore: lastmod changed from test

docsearch.json

@@ -0,0 +1,48 @@
+{
+  "index_name": "plural_docs",
+  "start_urls": [
+    "https://docs.plural.sh/"
+  ],
+  "stop_urls": [
+    "/pages-archive/",
+    "/__components/"
+  ],
+  "selectors": {
+    "lvl0": {
+      "selector": "h1",
+      "global": true,
+      "default_value": "Documentation"
+    },
+    "lvl1": "article h2",
+    "lvl2": "article h3",
+    "lvl3": "article h4",
+    "lvl4": "article h5",
+    "lvl5": "article h6",
+    "text": "article p, article li, article td",
+    "language": {
+      "selector": "/html/@lang",
+      "type": "xpath",
+      "global": true
+    }
+  },
+  "strip_chars": "0123456789-",
+  "custom_settings": {
+    "separatorsToIndex": "_",
+    "attributesForFaceting": ["language", "version"],
+    "attributesToRetrieve": [
+      "hierarchy",
+      "content",
+      "anchor",
+      "url",
+      "url_without_anchor",
+      "type"
+    ]
+  },
+  "js_render": false,
+  "use_anchors": true,
+  "selectors_exclude": [
+    ".DocSearch-content"
+  ],
+  "scrape_start_urls": true,
+  "min_indexed_level": 0
+} 

index-pages.mjs

@@ -1,4 +1,4 @@
-import { Dirent } from 'fs'
+import { execSync } from 'child_process'
 import { readdir, writeFile } from 'fs/promises'
 import path from 'path'
 import { fileURLToPath } from 'url'
@@ -7,23 +7,6 @@ const __filename = fileURLToPath(import.meta.url)
 
 const __dirname = path.dirname(__filename)
 
-function writeUrl({ location, lastMod, priority = '0.5' }) {
-  return `  <url>
-    <loc>${process.env.NEXT_PUBLIC_ROOT_URL}/${location}</loc>
-    <lastmod>${lastMod}</lastmod>
-    <changefreq>weekly</changefreq>
-    <priority>${priority || '0.5'}</priority>
-  </url>`
-}
-
-function wrapSiteMap(content) {
-  return `<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-${content}
-</urlset>
-`
-}
-
 const ignore = [
   // sitemaps
   /^sitemap\.xml.*/,
@@ -32,7 +15,7 @@ const ignore = [
   // [nextjs_dynamic_pages]
   // 404 and 500 error pages
   /^([_.[]|404|500).*$/,
-  /^__components\/.*/
+  /^__components\/.*/,
 ]
 
 const pageFilter = (file) => {
@@ -41,10 +24,41 @@ const pageFilter = (file) => {
       return false
     }
   }
+
   if (file?.isDirectory()) {
     return true
   }
-  return file.name.match(/\.(ts|tsx|js|jsx|md|mdoc)$/)
+
+  // Only include markdown files
+  return file.name.match(/\.(md|mdoc)$/)
+}
+
+// Function to strip numeric prefixes from path segments
+function stripNumericPrefixes(pathStr) {
+  return pathStr
+    .split('/')
+    .map((segment) => segment.replace(/^\d+-/, ''))
+    .join('/')
+}
+
+// Function to get the last git modification date of a file
+function getGitLastModified(filePath) {
+  try {
+    // Get the last commit date that modified this file in UTC
+    const date = execSync(`git log -1 --format="%cI" -- "${filePath}"`, {
+      encoding: 'utf-8',
+    }).trim()
+
+    if (!date) {
+      return new Date().toISOString()
+    }
+
+    // Convert to UTC and format as ISO string
+    return new Date(date).toISOString()
+  } catch (error) {
+    // Fallback to current date if git command fails
+    return new Date().toISOString()
+  }
 }
 
 const rootDir = __dirname
@@ -58,13 +72,25 @@ async function crawlPages(filePath = '/') {
   const filteredFiles = files.filter(pageFilter)
 
   const pages = []
+
   for (const file of filteredFiles) {
     if (file.isDirectory()) {
       pages.push(...(await crawlPages(path.join(filePath, file.name))))
     } else {
       let pathname = file.name.split('.').slice(0, -1).join('.')
+
       pathname = path.join(filePath, pathname.replace(/(^|\/)index$/g, ''))
-      pages.push({ path: pathname })
+      // Strip numeric prefixes from the entire path
+      pathname = stripNumericPrefixes(pathname)
+
+      // Get git modification time
+      const fullFilePath = path.join(fullPath, file.name)
+      const lastmod = getGitLastModified(fullFilePath)
+
+      pages.push({
+        path: pathname,
+        lastmod,
+      })
     }
   }
 

next.config.js

@@ -27,6 +27,11 @@ const nextConfig = {
   },
   async redirects() {
     return [
+      {
+        source: '/getting-started/agent-api-reference',
+        destination: '/overview/agent-api-reference',
+        permanent: true,
+      },
       {
         source: '/getting-started/readme',
         destination: '/',
@@ -43,200 +48,15 @@ const nextConfig = {
         permanent: true,
       },
       {
-        source: '/reference/architecture-1',
-        destination: '/reference/architecture',
-        permanent: true,
-      },
-      {
-        source: '/reference/workspaces/workspace-structure',
-        destination: '/reference/workspaces',
-        permanent: true,
-      },
-      {
-        source: '/getting-started/getting-started-with-runbooks/runbook-yaml',
-        destination:
-          '/adding-new-application/getting-started-with-runbooks/runbook-yaml',
-        permanent: true,
-      },
-      {
-        source: '/basic-setup-and-deployment/setting-up-gitops',
-        destination:
-          '/getting-started/managing-git-repository/setting-up-gitops',
-        permanent: true,
-      },
-      {
-        source: '/basic-setup-and-deployment/openid-connect',
-        destination: '/getting-started/openid-connect',
-        permanent: true,
-      },
-      {
-        source: '/basic-setup-and-deployment/admin-console',
-        destination: '/getting-started/admin-console',
-        permanent: true,
-      },
-      {
-        source: '/basic-setup-and-deployment/cloud-shell-quickstart',
-        destination: '/getting-started/cloud-shell-quickstart',
-        permanent: true,
-      },
-      {
-        source: '/basic-setup-and-deployment/uninstall',
-        destination: '/operations/uninstall',
-        permanent: true,
-      },
-      {
-        source: '/reference/operator-guides',
-        destination: '/repositories',
-        permanent: true,
-      },
-      {
-        source: '/reference/operator-guides/adding-kubecost-for-cost-analysis',
-        destination: '/repositories/kubecost',
+        source: '/deployments/architecture',
+        destination: '/overview/architecture',
         permanent: true,
       },
       {
         source: '/reference/architecture',
         destination: '/',
         permanent: true,
       },
-      {
-        source: '/repositories',
-        destination: '/applications',
-        permanent: true,
-      },
-      {
-        source: '/repositories/airbyte',
-        destination: '/applications/airbyte',
-        permanent: true,
-      },
-      {
-        source: '/repositories/airflow',
-        destination: '/applications/airflow',
-        permanent: true,
-      },
-      {
-        source: '/repositories/console',
-        destination: '/applications/console',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics',
-        destination: '/operations',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/network-configuration',
-        destination: '/operations/network-configuration',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/dns-setup',
-        destination: '/operations/dns-setup',
-        permanent: true,
-      },
-      {
-        source:
-          '/advanced-topics/dns-setup/creating-dns-zone-in-your-cloud-provider-console',
-        destination:
-          '/operations/dns-setup/creating-dns-zone-in-your-cloud-provider-console',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/security',
-        destination: '/operations/security',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/security/secret-management',
-        destination:
-          '/getting-started/manage-git-repositories/sharing-git-repositories',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/debugging',
-        destination: '/debugging',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/debugging/health-checks',
-        destination: '/debugging/health-checks',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/debugging/logs',
-        destination: '/debugging/logs',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/debugging/proxies',
-        destination: '/debugging/proxies',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/identity-and-access-management',
-        destination: '/operations/auth-access-control',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/identity-and-access-management/introduction',
-        destination: '/operations/auth-access-control',
-        permanent: true,
-      },
-      {
-        source:
-          '/advanced-topics/identity-and-access-management/openid-connect',
-        destination: '/operations/auth-access-control/openid-connect',
-        permanent: true,
-      },
-      {
-        source: '/advanced-topics/identity-and-access-management/api-tokens',
-        destination: '/operations/auth-access-control/api-tokens',
-        permanent: true,
-      },
-      {
-        source:
-          '/advanced-topics/identity-and-access-management/identity-and-installations',
-        destination:
-          '/operations/auth-access-control/identity-and-installations',
-        permanent: true,
-      },
-      {
-        source:
-          '/advanced-topics/identity-and-access-management/identity-and-installations/audit-logging',
-        destination:
-          '/operations/auth-access-control/identity-and-installations/audit-logging',
-        permanent: true,
-      },
-      {
-        source:
-          '/advanced-topics/identity-and-access-management/identity-and-installations/service-accounts',
-        destination:
-          '/operations/auth-access-control/identity-and-installations/service-accounts',
-        permanent: true,
-      },
-      {
-        source:
-          '/advanced-topics/identity-and-access-management/identity-and-installations/sharing-existing-repos',
-        destination:
-          '/getting-started/manage-git-repositories/sharing-git-repository',
-        permanent: true,
-      },
-      {
-        source: '/reference/workspaces',
-        destination:
-          '/getting-started/manage-git-repositories/your-plural-workspace',
-        permanent: true,
-      },
-      {
-        source: '/introduction',
-        destination: '/overview/introduction',
-        permanent: true,
-      },
-      {
-        source: '/deployments/architecture',
-        destination: '/overview/architecture',
-        permanent: true,
-      },
       {
         source: '/deployments/operator/api',
         destination: '/overview/api-reference',

package.json

@@ -4,9 +4,9 @@
     "node": "22.12.0"
   },
   "scripts": {
-    "dev": "run-s generate:routes && concurrently \"next dev\" \"yarn graphql:codegen:watch\"",
+    "dev": "run-s generate:routes generate:pages track:redirects && concurrently \"next dev\" \"yarn graphql:codegen:watch\"",
     "dev:watch": "concurrently \"next dev\" \"yarn graphql:codegen:watch\"",
-    "build": "run-s generate:routes build:next",
+    "build": "run-s generate:routes generate:pages build:next",
     "build:next": "next build",
     "start": "next start",
     "lint": "run-p lint:format lint:js lint:css",
@@ -19,7 +19,9 @@
     "fix:css": "stylelint src/**/*.css --fix",
     "graphql:codegen": "graphql-codegen --config codegen.yml",
     "graphql:codegen:watch": "graphql-codegen --watch --config codegen.yml",
-    "generate:routes": "ts-node scripts/generate-unified-routes.ts"
+    "generate:routes": "ts-node scripts/generate-unified-routes.ts",
+    "generate:pages": "node index-pages.mjs",
+    "track:redirects": "ts-node scripts/track-moves.ts"
   },
   "dependencies": {
     "@apollo/client": "3.7.15",