feat: add SEO and AI optimization

- Add llms.txt for AI crawler discoverability
- Add robots.txt with AI crawler permissions
- Update VitePress config with:
  - Open Graph meta tags
  - Twitter card meta tags
  - JSON-LD structured data
  - Sitemap generation
  - SEO keywords and description
- Add branded OG image for social sharing

Author: aminembarki

.vitepress/config.ts

@@ -2,9 +2,67 @@ import { defineConfig } from 'vitepress';
 
 export default defineConfig({
   title: 'Crawlee Cloud',
-  description: 'Open-source self-hosted platform for Crawlee scrapers',
+  description:
+    'Self-hosted, open-source platform for running Crawlee and Apify Actors on your own infrastructure. Deploy web scrapers with Playwright, Puppeteer, or Cheerio.',
   cleanUrls: true,
+  lang: 'en-US',
+  lastUpdated: true,
+
+  // Sitemap generation
+  sitemap: {
+    hostname: 'https://crawlee.cloud',
+    lastmodDateOnly: false,
+  },
+
   head: [
+    // Primary Meta Tags
+    ['meta', { name: 'viewport', content: 'width=device-width, initial-scale=1.0' }],
+    [
+      'meta',
+      {
+        name: 'keywords',
+        content:
+          'crawlee, apify, web scraping, self-hosted, open source, playwright, puppeteer, cheerio, web crawler, data extraction, scraping platform, actor platform',
+      },
+    ],
+    ['meta', { name: 'author', content: 'Crawlee Cloud' }],
+    ['meta', { name: 'robots', content: 'index, follow' }],
+    ['link', { rel: 'canonical', href: 'https://crawlee.cloud' }],
+
+    // Open Graph / Facebook
+    ['meta', { property: 'og:type', content: 'website' }],
+    ['meta', { property: 'og:url', content: 'https://crawlee.cloud' }],
+    ['meta', { property: 'og:title', content: 'Crawlee Cloud - Your Scrapers, Your Cloud' }],
+    [
+      'meta',
+      {
+        property: 'og:description',
+        content:
+          'Self-hosted, open-source platform for running Crawlee and Apify Actors on your own infrastructure.',
+      },
+    ],
+    ['meta', { property: 'og:image', content: 'https://crawlee.cloud/og-image.png' }],
+    ['meta', { property: 'og:site_name', content: 'Crawlee Cloud' }],
+    ['meta', { property: 'og:locale', content: 'en_US' }],
+
+    // Twitter / X
+    ['meta', { name: 'twitter:card', content: 'summary_large_image' }],
+    ['meta', { name: 'twitter:url', content: 'https://crawlee.cloud' }],
+    ['meta', { name: 'twitter:title', content: 'Crawlee Cloud - Your Scrapers, Your Cloud' }],
+    [
+      'meta',
+      {
+        name: 'twitter:description',
+        content:
+          'Self-hosted, open-source platform for running Crawlee and Apify Actors on your own infrastructure.',
+      },
+    ],
+    ['meta', { name: 'twitter:image', content: 'https://crawlee.cloud/og-image.png' }],
+
+    // Favicons (if you add them later)
+    ['link', { rel: 'icon', type: 'image/svg+xml', href: '/logo.svg' }],
+
+    // Google Analytics
     [
       'script',
       { async: '', src: 'https://www.googletagmanager.com/gtag/js?id=G-FYMRJ2GG39' },
@@ -17,6 +75,37 @@ export default defineConfig({
       gtag('js', new Date());
       gtag('config', 'G-FYMRJ2GG39');`,
     ],
+
+    // JSON-LD Structured Data
+    [
+      'script',
+      { type: 'application/ld+json' },
+      JSON.stringify({
+        '@context': 'https://schema.org',
+        '@type': 'SoftwareApplication',
+        name: 'Crawlee Cloud',
+        applicationCategory: 'DeveloperApplication',
+        operatingSystem: 'Linux, macOS, Windows',
+        description:
+          'Self-hosted, open-source platform for running Crawlee and Apify Actors on your own infrastructure.',
+        url: 'https://crawlee.cloud',
+        offers: {
+          '@type': 'Offer',
+          price: '0',
+          priceCurrency: 'USD',
+        },
+        author: {
+          '@type': 'Organization',
+          name: 'Crawlee Cloud',
+          url: 'https://crawlee.cloud',
+        },
+        license: 'https://opensource.org/licenses/MIT',
+        codeRepository: 'https://github.com/crawlee-cloud/crawlee-cloud',
+        programmingLanguage: ['TypeScript', 'JavaScript'],
+        keywords:
+          'web scraping, crawlee, apify, self-hosted, open source, playwright, puppeteer, cheerio',
+      }),
+    ],
   ],
   srcDir: 'src',
   outDir: './dist',

dist/404.html

@@ -2,21 +2,42 @@
 <html lang="en-US" dir="ltr">
   <head>
     <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width,initial-scale=1">
+    
     <title>404 | Crawlee Cloud</title>
     <meta name="description" content="Not Found">
     <meta name="generator" content="VitePress v1.6.4">
-    <link rel="preload stylesheet" href="/assets/style.BWDTHy11.css" as="style">
+    <link rel="preload stylesheet" href="/assets/style.B1qQlgUi.css" as="style">
     <link rel="preload stylesheet" href="/vp-icons.css" as="style">
 
-    <script type="module" src="/assets/app.BOliJLSL.js"></script>
+    <script type="module" src="/assets/app.D2DgKinZ.js"></script>
     <link rel="preload" href="/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="keywords" content="crawlee, apify, web scraping, self-hosted, open source, playwright, puppeteer, cheerio, web crawler, data extraction, scraping platform, actor platform">
+    <meta name="author" content="Crawlee Cloud">
+    <meta name="robots" content="index, follow">
+    <link rel="canonical" href="https://crawlee.cloud">
+    <meta property="og:type" content="website">
+    <meta property="og:url" content="https://crawlee.cloud">
+    <meta property="og:title" content="Crawlee Cloud - Your Scrapers, Your Cloud">
+    <meta property="og:description" content="Self-hosted, open-source platform for running Crawlee and Apify Actors on your own infrastructure.">
+    <meta property="og:image" content="https://crawlee.cloud/og-image.png">
+    <meta property="og:site_name" content="Crawlee Cloud">
+    <meta property="og:locale" content="en_US">
+    <meta name="twitter:card" content="summary_large_image">
+    <meta name="twitter:url" content="https://crawlee.cloud">
+    <meta name="twitter:title" content="Crawlee Cloud - Your Scrapers, Your Cloud">
+    <meta name="twitter:description" content="Self-hosted, open-source platform for running Crawlee and Apify Actors on your own infrastructure.">
+    <meta name="twitter:image" content="https://crawlee.cloud/og-image.png">
+    <link rel="icon" type="image/svg+xml" href="/logo.svg">
+    <script async src="https://www.googletagmanager.com/gtag/js?id=G-FYMRJ2GG39"></script>
+    <script>window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-FYMRJ2GG39");</script>
+    <script type="application/ld+json">{"@context":"https://schema.org","@type":"SoftwareApplication","name":"Crawlee Cloud","applicationCategory":"DeveloperApplication","operatingSystem":"Linux, macOS, Windows","description":"Self-hosted, open-source platform for running Crawlee and Apify Actors on your own infrastructure.","url":"https://crawlee.cloud","offers":{"@type":"Offer","price":"0","priceCurrency":"USD"},"author":{"@type":"Organization","name":"Crawlee Cloud","url":"https://crawlee.cloud"},"license":"https://opensource.org/licenses/MIT","codeRepository":"https://github.com/crawlee-cloud/crawlee-cloud","programmingLanguage":["TypeScript","JavaScript"],"keywords":"web scraping, crawlee, apify, self-hosted, open source, playwright, puppeteer, cheerio"}</script>
     <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
     <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
   </head>
   <body>
     <div id="app"></div>
-    <script>window.__VP_HASH_MAP__=JSON.parse("{\"docs_api.md\":\"D3YRveVj\",\"docs_apify-sdk-environment.md\":\"B5PGvbGR\",\"docs_cli.md\":\"uL6wdgO-\",\"docs_dashboard.md\":\"BisIJXWe\",\"docs_deployment.md\":\"qTpXi688\",\"docs_index.md\":\"w1F42R42\",\"docs_runner.md\":\"BGocZfZH\",\"index.md\":\"CPBbALSI\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Crawlee Cloud\",\"description\":\"Open-source self-hosted platform for Crawlee scrapers\",\"base\":\"/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"siteTitle\":false,\"logo\":\"/logo.svg\",\"nav\":[{\"text\":\"Home\",\"link\":\"/\"},{\"text\":\"Docs\",\"link\":\"/docs/\"},{\"text\":\"GitHub\",\"link\":\"https://github.com/crawlee-cloud/crawlee-cloud\"}],\"sidebar\":[{\"text\":\"Introduction\",\"items\":[{\"text\":\"Overview\",\"link\":\"/docs/\"},{\"text\":\"Apify SDK Environment\",\"link\":\"/docs/apify-sdk-environment\"}]},{\"text\":\"Components\",\"items\":[{\"text\":\"CLI\",\"link\":\"/docs/cli\"},{\"text\":\"Dashboard\",\"link\":\"/docs/dashboard\"},{\"text\":\"Runner\",\"link\":\"/docs/runner\"},{\"text\":\"Deployment\",\"link\":\"/docs/deployment\"},{\"text\":\"API Reference\",\"link\":\"/docs/api\"}]}],\"socialLinks\":[{\"icon\":\"github\",\"link\":\"https://github.com/crawlee-cloud/crawlee-cloud\"}],\"footer\":{\"message\":\"Released under the MIT License.\",\"copyright\":\"Copyright © 2025 Crawlee Cloud\"},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":true}");</script>
+    <script>window.__VP_HASH_MAP__=JSON.parse("{\"docs_api.md\":\"BIXT62I7\",\"docs_apify-sdk-environment.md\":\"DvYz4nYC\",\"docs_cli.md\":\"Centaw9h\",\"docs_dashboard.md\":\"DHYgmMGv\",\"docs_deployment.md\":\"Cg_aVGTo\",\"docs_index.md\":\"C4xmGzKy\",\"docs_runner.md\":\"BgQxNnu7\",\"index.md\":\"NJbyKGSV\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Crawlee Cloud\",\"description\":\"Self-hosted, open-source platform for running Crawlee and Apify Actors on your own infrastructure. Deploy web scrapers with Playwright, Puppeteer, or Cheerio.\",\"base\":\"/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"siteTitle\":false,\"logo\":{\"light\":\"/logo-light.svg\",\"dark\":\"/logo-dark.svg\"},\"nav\":[{\"text\":\"Home\",\"link\":\"/\"},{\"text\":\"Docs\",\"link\":\"/docs/\"},{\"text\":\"GitHub\",\"link\":\"https://github.com/crawlee-cloud/crawlee-cloud\"}],\"sidebar\":[{\"text\":\"Introduction\",\"items\":[{\"text\":\"Overview\",\"link\":\"/docs/\"},{\"text\":\"Apify SDK Environment\",\"link\":\"/docs/apify-sdk-environment\"}]},{\"text\":\"Components\",\"items\":[{\"text\":\"CLI\",\"link\":\"/docs/cli\"},{\"text\":\"Dashboard\",\"link\":\"/docs/dashboard\"},{\"text\":\"Runner\",\"link\":\"/docs/runner\"},{\"text\":\"Deployment\",\"link\":\"/docs/deployment\"},{\"text\":\"API Reference\",\"link\":\"/docs/api\"}]}],\"socialLinks\":[{\"icon\":\"github\",\"link\":\"https://github.com/crawlee-cloud/crawlee-cloud\"}],\"footer\":{\"message\":\"Released under the MIT License.\",\"copyright\":\"Copyright © 2025 Crawlee Cloud\"},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":true}");</script>
 
   </body>
 </html>