Improve Agent Readiness

Author: rcosta358

.well-known/api-catalog

@@ -0,0 +1,25 @@
+---
+layout: null
+permalink: /.well-known/api-catalog
+---
+{
+  "name": {{ site.title | jsonify }},
+  "description": "Public documentation for LiquidJava.",
+  "links": [
+    {
+      "rel": "service-doc",
+      "href": "{{ '/resources/' | absolute_url }}",
+      "type": "text/html"
+    },
+    {
+      "rel": "sitemap",
+      "href": "{{ '/sitemap.xml' | absolute_url }}",
+      "type": "application/xml"
+    },
+    {
+      "rel": "robots",
+      "href": "{{ '/robots.txt' | absolute_url }}",
+      "type": "text/plain"
+    }
+  ]
+}

_config.yml

@@ -13,6 +13,9 @@ baseurl: /liquidjava-docs
 permalink: pretty
 search_enabled: true
 heading_anchors: true
+include:
+  - .well-known
+  - _headers
 
 favicon_ico: "/assets/images/favicon.ico"
 

_headers

@@ -0,0 +1,13 @@
+---
+layout: null
+permalink: /_headers
+---
+/
+  Link: <{{ '/.well-known/api-catalog' | relative_url }}>; rel="api-catalog"; type="application/json"
+  Link: <{{ '/resources/' | relative_url }}>; rel="service-doc"; type="text/html"
+  Link: <{{ '/sitemap.xml' | relative_url }}>; rel="sitemap"; type="application/xml"
+
+{{ site.baseurl }}/
+  Link: <{{ '/.well-known/api-catalog' | relative_url }}>; rel="api-catalog"; type="application/json"
+  Link: <{{ '/resources/' | relative_url }}>; rel="service-doc"; type="text/html"
+  Link: <{{ '/sitemap.xml' | relative_url }}>; rel="sitemap"; type="application/xml"

robots.txt

@@ -0,0 +1,28 @@
+---
+layout: null
+permalink: /robots.txt
+---
+# robots.txt for {{ site.title }}
+# See RFC 9309: https://www.rfc-editor.org/rfc/rfc9309
+
+User-agent: *
+Allow: {{ site.baseurl }}/
+Disallow: {{ site.baseurl }}/404.html
+
+# AI search and answer agents are allowed to retrieve public documentation.
+User-agent: OAI-SearchBot
+Allow: {{ site.baseurl }}/
+
+User-agent: Claude-Web
+Allow: {{ site.baseurl }}/
+
+# Training-focused crawlers are not authorized to use this site for model training.
+User-agent: GPTBot
+Disallow: {{ site.baseurl }}/
+
+User-agent: Google-Extended
+Disallow: {{ site.baseurl }}/
+
+Content-Signal: ai-train=no, search=yes, ai-input=yes
+
+Sitemap: {{ site.url }}{{ site.baseurl }}/sitemap.xml

sitemap.xml

@@ -0,0 +1,16 @@
+---
+layout: null
+permalink: /sitemap.xml
+---
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  {%- assign sitemap_pages = site.pages | where_exp: "page", "page.path contains '.md'" | where_exp: "page", "page.url != '/404.html'" | where_exp: "page", "page.sitemap != false" | sort: "url" -%}
+  {%- for page in sitemap_pages %}
+  <url>
+    <loc>{{ page.url | absolute_url | xml_escape }}</loc>
+    {%- if page.last_modified_at %}
+    <lastmod>{{ page.last_modified_at | date_to_xmlschema }}</lastmod>
+    {%- endif %}
+  </url>
+  {%- endfor %}
+</urlset>