fix(seo): use canonical SITE_URL for robots and sitemap (#4598)

* fix(seo): use canonical SITE_URL for robots and sitemap

* fix(seo): drop /templates from sitemap and guard robots/sitemap in seo test

Author: waleedlatif1

apps/sim/app/(landing)/seo.test.ts

@@ -26,9 +26,21 @@ const SEO_SCAN_DIRS = [
 
 const SEO_SCAN_INDIVIDUAL_FILES = [
   path.resolve(APP_DIR, 'page.tsx'),
+  path.resolve(APP_DIR, 'robots.ts'),
+  path.resolve(APP_DIR, 'sitemap.ts'),
   path.resolve(SIM_ROOT, 'ee', 'whitelabeling', 'metadata.ts'),
 ]
 
+/**
+ * Files whose entire URL output is SEO-facing (robots.txt, sitemap.xml).
+ * Unlike metadata exports, these don't use `metadataBase`, so the existing
+ * `getBaseUrl()`-in-metadata check would miss a regression here.
+ */
+const SEO_DEFAULT_EXPORT_FILES = [
+  path.resolve(APP_DIR, 'robots.ts'),
+  path.resolve(APP_DIR, 'sitemap.ts'),
+]
+
 function collectFiles(dir: string, exts: string[]): string[] {
   const results: string[] = []
   if (!fs.existsSync(dir)) return results
@@ -97,6 +109,21 @@ describe('SEO canonical URLs', () => {
     ).toHaveLength(0)
   })
 
+  it('robots.ts and sitemap.ts do not import getBaseUrl', () => {
+    const violations: string[] = []
+    for (const file of SEO_DEFAULT_EXPORT_FILES) {
+      if (!fs.existsSync(file)) continue
+      const content = fs.readFileSync(file, 'utf-8')
+      if (content.includes('getBaseUrl')) {
+        violations.push(path.relative(SIM_ROOT, file))
+      }
+    }
+    expect(
+      violations,
+      `robots.ts/sitemap.ts must use SITE_URL, not getBaseUrl():\n${violations.join('\n')}`
+    ).toHaveLength(0)
+  })
+
   it('public pages do not use getBaseUrl() for SEO metadata', () => {
     const files = getAllSeoFiles(['.ts', '.tsx'])
     const violations: string[] = []

apps/sim/app/robots.ts

@@ -1,5 +1,5 @@
 import type { MetadataRoute } from 'next'
-import { getBaseUrl } from '@/lib/core/utils/urls'
+import { SITE_URL } from '@/lib/core/utils/urls'
 
 const DISALLOWED_PATHS = [
   '/api/',
@@ -45,8 +45,6 @@ const LINK_PREVIEW_BOTS = [
 ]
 
 export default function robots(): MetadataRoute.Robots {
-  const baseUrl = getBaseUrl()
-
   return {
     rules: [
       { userAgent: '*', allow: '/', disallow: DISALLOWED_PATHS },
@@ -56,6 +54,6 @@ export default function robots(): MetadataRoute.Robots {
         disallow: LINK_PREVIEW_DISALLOWED_PATHS,
       },
     ],
-    sitemap: [`${baseUrl}/sitemap.xml`, `${baseUrl}/blog/sitemap-images.xml`],
+    sitemap: [`${SITE_URL}/sitemap.xml`, `${SITE_URL}/blog/sitemap-images.xml`],
   }
 }

apps/sim/app/sitemap.ts

@@ -1,12 +1,12 @@
 import type { MetadataRoute } from 'next'
 import { COURSES } from '@/lib/academy/content'
 import { getAllPostMeta } from '@/lib/blog/registry'
-import { getBaseUrl } from '@/lib/core/utils/urls'
+import { SITE_URL } from '@/lib/core/utils/urls'
 import integrations from '@/app/(landing)/integrations/data/integrations.json'
 import { ALL_CATALOG_MODELS, MODEL_PROVIDERS_WITH_CATALOGS } from '@/app/(landing)/models/utils'
 
 export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
-  const baseUrl = getBaseUrl()
+  const baseUrl = SITE_URL
   const posts = await getAllPostMeta()
 
   const latestPostDate =
@@ -46,6 +46,9 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
     {
       url: `${baseUrl}/partners`,
     },
+    {
+      url: `${baseUrl}/contact`,
+    },
     {
       url: `${baseUrl}/terms`,
       lastModified: new Date('2024-10-14'),