diff --git a/cdn/docs_dev.tf b/cdn/docs_dev.tf index f506c53..166d9c6 100644 --- a/cdn/docs_dev.tf +++ b/cdn/docs_dev.tf @@ -1,6 +1,16 @@ +# The S3-backend canary for docs.ruby-lang.org. The production service keeps +# pointing at docs-origin until this proves out, then docs.tf adopts the same +# shape. Backend selection goes through request_conditions on a header flag the +# custom VCL sets before #FASTLY recv (assigning req.backend in VCL would +# bypass shielding, see cache.tf); the docs-origin backend stays as the +# fallback for unflagged requests so paths can be moved over one at a time. resource "fastly_service_vcl" "docs_dev" { activate = true stage = false + # A shielded fetch needs a Host that is a domain of this service, so the + # bucket endpoint doubles as default_host and as a domain below, the same + # arrangement as cache.tf. The other backends override_host instead. + default_host = "docs.r-l.o.s3.amazonaws.com" default_ttl = 60 http3 = true name = "docs-dev.ruby-lang.org" @@ -19,8 +29,10 @@ resource "fastly_service_vcl" "docs_dev" { max_lifetime = 0 max_use = 0 name = "docs origin server" + override_host = "docs.ruby-lang.org" port = 443 prefer_ipv6 = false + request_condition = "backend-is-origin" shield = "tyo-tokyo-jp" ssl_cert_hostname = "docs-origin.ruby-lang.org" ssl_check_cert = true @@ -28,6 +40,77 @@ resource "fastly_service_vcl" "docs_dev" { weight = 100 } + # The docs bucket (aws_s3_bucket.docs in s3.tf): public/ root files, en and + # ja. us-east-1, so shielded near it like the cache service's S3 backend. + backend { + address = "s3.amazonaws.com" + auto_loadbalance = false + between_bytes_timeout = 10000 + connect_timeout = 1000 + error_threshold = 0 + first_byte_timeout = 15000 + keepalive_time = 0 + max_conn = 200 + max_lifetime = 0 + max_use = 0 + name = "s3-docs" + port = 443 + prefer_ipv6 = false + request_condition = "backend-is-docs-s3" + shield = "iad-va-us" + ssl_cert_hostname = "s3.amazonaws.com" + ssl_check_cert = true + use_ssl = true + weight = 100 + } + + # Doxygen stays in the rubyci bucket for now (step 1 of the migration); + # dropping this backend and pointing doxygen.yml at the docs bucket's + # capi/en/master/ prefix is step 2. The bucket name has no dots, so the + # virtual-hosted endpoint works as the TLS hostname directly. Unshielded: + # it refreshes every three hours and carries little traffic. + backend { + address = "rubyci.s3.amazonaws.com" + auto_loadbalance = false + between_bytes_timeout = 10000 + connect_timeout = 1000 + error_threshold = 0 + first_byte_timeout = 15000 + keepalive_time = 0 + max_conn = 200 + max_lifetime = 0 + max_use = 0 + name = "s3-rubyci-doxygen" + port = 443 + prefer_ipv6 = false + request_condition = "backend-is-doxygen-s3" + ssl_cert_hostname = "rubyci.s3.amazonaws.com" + ssl_check_cert = true + use_ssl = true + weight = 100 + } + + condition { + name = "backend-is-origin" + priority = 10 + statement = "!req.http.X-Docs-Backend" + type = "REQUEST" + } + + condition { + name = "backend-is-docs-s3" + priority = 10 + statement = "req.http.X-Docs-Backend == \"s3\"" + type = "REQUEST" + } + + condition { + name = "backend-is-doxygen-s3" + priority = 10 + statement = "req.http.X-Docs-Backend == \"doxygen\"" + type = "REQUEST" + } + # A shielded miss runs the logging endpoint at both POPs, so one request # becomes two events carrying the same byte count. The edge sets Fastly-FF when # it forwards to the shield, so this keeps the edge line, which is the one with @@ -39,6 +122,18 @@ resource "fastly_service_vcl" "docs_dev" { type = "RESPONSE" } + # What /ja/latest and /ja/master resolve to (the bucket holds no symlink + # objects), and what the unreleased-version redirects key off. A release + # updates these values and everything else follows. + dictionary { + name = "docs_versions" + } + + domain { + comment = "For shielding" + name = "docs.r-l.o.s3.amazonaws.com" + } + # Reached only through the Fastly-provided domain, same as cache-dev. That # needs neither a ruby-lang.org zone change nor a TLS subscription, since the # shared certificate already covers it. @@ -47,8 +142,8 @@ resource "fastly_service_vcl" "docs_dev" { } gzip { - content_types = ["text/html", "application/x-javascript", "text/css", "application/javascript", "text/javascript", "application/json", "application/vnd.ms-fontobject", "application/x-font-opentype", "application/x-font-truetype", "application/x-font-ttf", "application/xml", "font/eot", "font/opentype", "font/otf", "image/svg+xml", "image/vnd.microsoft.icon", "text/plain", "text/xml"] - extensions = ["css", "js", "html", "eot", "ico", "otf", "ttf", "json", "svg"] + content_types = ["text/html", "application/x-javascript", "text/css", "application/javascript", "text/javascript", "application/json", "application/vnd.ms-fontobject", "application/x-font-opentype", "application/x-font-truetype", "application/x-font-ttf", "application/xml", "font/eot", "font/opentype", "font/otf", "image/svg+xml", "image/vnd.microsoft.icon", "text/plain", "text/xml", "text/markdown"] + extensions = ["css", "js", "html", "eot", "ico", "otf", "ttf", "json", "svg", "md", "xml", "txt"] name = "Default Gzip Policy" } @@ -72,12 +167,19 @@ resource "fastly_service_vcl" "docs_dev" { xff = "append" } - # Production docs intentionally has no custom VCL. This file is the - # behavior-neutral boilerplate equivalent, uploaded so VCL-level changes can - # be canaried on docs-dev before deciding how to apply them to production. vcl { content = file("${path.module}/vcl/docs_dev.vcl") main = true name = "default" } } + +resource "fastly_service_dictionary_items" "docs_dev_versions" { + service_id = fastly_service_vcl.docs_dev.id + dictionary_id = one([for d in fastly_service_vcl.docs_dev.dictionary : d.dictionary_id if d.name == "docs_versions"]) + + items = { + latest = "4.0" + master = "4.1" + } +} diff --git a/cdn/s3.tf b/cdn/s3.tf index da1179a..57ab39b 100644 --- a/cdn/s3.tf +++ b/cdn/s3.tf @@ -86,6 +86,90 @@ resource "aws_s3_bucket_lifecycle_configuration" "ftp" { } } +# Origin of the docs service (docs.ruby-lang.org), canaried on docs-dev first. +# Prefixes and their writers: +# (root) -- ruby/docs.ruby-lang.org public/ (robots.txt, llms.txt, sitemap.xml, index pages, assets) +# en// -- ruby/actions docs.yml (extracted RDoc HTML; frozen 3.0/3.1 synced once by hand) +# ja// -- rurema/generated-documents html/ja/* (latest/master are resolved at the +# edge from the docs_versions dictionary; no symlink objects in the bucket) +# capi/en/master/ -- ruby/actions doxygen.yml, once it moves off the rubyci bucket +# us-east-1 like ftp.r-l.o: every request comes through Fastly, so client +# latency does not depend on the bucket region and the cheapest tier wins. +resource "aws_s3_bucket" "docs" { + bucket = "docs.r-l.o" + region = "us-east-1" + + tags = { + Name = "docs.r-l.o" + } + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_s3_bucket_policy" "docs" { + bucket = aws_s3_bucket.docs.bucket + region = "us-east-1" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "Allow Public Access to All Objects" + Effect = "Allow" + Principal = "*" + Action = ["s3:GetObject", "s3:GetObjectTagging"] + Resource = ["arn:aws:s3:::docs.r-l.o", "arn:aws:s3:::docs.r-l.o/*"] + }, + ] + }) +} + +resource "aws_s3_bucket_versioning" "docs" { + bucket = aws_s3_bucket.docs.bucket + region = "us-east-1" + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "docs" { + bucket = aws_s3_bucket.docs.bucket + region = "us-east-1" + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "docs" { + bucket = aws_s3_bucket.docs.bucket + region = "us-east-1" + + transition_default_minimum_object_size = "varies_by_storage_class" + + rule { + id = "lifecycle" + status = "Enabled" + + expiration { + expired_object_delete_marker = true + } + + noncurrent_version_expiration { + noncurrent_days = 7 + } + + abort_incomplete_multipart_upload { + days_after_initiation = 1 + } + } +} + # Origin of the logs.rubyci.org service (chkbuild logs). resource "aws_s3_bucket" "rubyci" { bucket = "rubyci" diff --git a/cdn/vcl/docs_dev.vcl b/cdn/vcl/docs_dev.vcl index 7f4aa3f..b6d317e 100644 --- a/cdn/vcl/docs_dev.vcl +++ b/cdn/vcl/docs_dev.vcl @@ -1,4 +1,135 @@ sub vcl_recv { + + # ---- S3 backend routing and rewrites (docs-dev canary) ---- + # Runs before #FASTLY recv so the generated backend-selection code + # (request_condition) can see the X-Docs-Backend flag, same as cache.vcl. + # Client-visible URLs never change here: everything except the synthetic + # redirects (error 601/602) is an internal rewrite, and the cache key is + # the rewritten URL. + + declare local var.lang STRING; + declare local var.ver STRING; + declare local var.rest STRING; + + if (req.request == "HEAD" || req.request == "GET") { + + # Surrogate-Key and redirects are computed from what the client asked + # for, so keep the pre-rewrite URL. It survives restarts (markdown + # fallback below), so only set it on the first pass. + if (!req.http.X-Orig-Url) { + set req.http.X-Orig-Url = req.url; + } + + # Vary: Accept is served on the negotiable /ja/ pages below. Normalize + # Accept to two values first so the variants cannot explode per client. + if (req.url ~ "^/ja/") { + if (req.http.Accept ~ "text/markdown") { + set req.http.Accept = "text/markdown"; + } else { + unset req.http.Accept; + } + } + + # ---- redirects, same rules and order as the nginx origin ---- + if (req.url ~ "^/en/trunk(.*)$") { + set req.http.X-Redirect-Location = "https://docs.ruby-lang.org/en/master" re.group.1; + error 601; + } + # https://github.com/ruby/docs.ruby-lang.org/issues/130 + if (req.url ~ "^/en/([^/]+)/doc/(.*)$") { + set req.http.X-Redirect-Location = "/en/" re.group.1 "/" re.group.2; + error 601; + } + # 2.8.0 was renamed to 3.0.0, and the directory is 3.0 + if (req.url ~ "^/(en|ja)/(2\.8\.0|3\.0\.0)(.*)$") { + set req.http.X-Redirect-Location = "/" re.group.1 "/3.0" re.group.3; + error 601; + } + # Old rurema-search /ja/search/query:WORD/ URLs; ?q= keeps the + # percent-encoding because req.url is still encoded here. + if (req.url ~ "^/ja/search/" && req.url ~ "query:") { + if (req.url ~ "^/ja/search/(?:[^?]*?/)??query:([^/?]+)") { + set req.http.X-Redirect-Location = "/ja/search/?q=" re.group.1; + } else { + set req.http.X-Redirect-Location = "/ja/search/"; + } + error 601; + } + + # The unreleased version is not public until the release: ja/master + # pages point at en/ through their [rdoc] links (which only + # exists as en/master), and ja/ itself is not linked from the + # version index. Both go to the master alias with a temporary redirect + # that disappears once the docs_versions dictionary moves at release. + if (req.url ~ "^/(en|ja)/([^/?]+)(/[^?]*)?$") { + set var.lang = re.group.1; + set var.ver = re.group.2; + set var.rest = re.group.3; + if (var.ver == table.lookup(docs_versions, "master")) { + if (var.rest == "") { + set var.rest = "/"; + } + set req.http.X-Redirect-Location = "/" var.lang "/master" var.rest; + error 602; + } + } + + # S3 interprets query strings as API parameters and the static site + # never varies on them (/ja/search/?q= is read by client-side JS), so + # drop them from the cache key and the backend request. + set req.url = req.url.path; + + # Directory-looking URL without the trailing slash: redirect to the + # slash form, like nginx did for directories. Every real page has an + # extension, so no dot in the last segment is a safe heuristic. + if (req.url ~ "^/(en|ja|capi)(/|$)" && req.url !~ "\.[^/]+$" && req.url !~ "/$") { + set req.http.X-Redirect-Location = req.url "/"; + error 601; + } + + # /ja/latest and /ja/master have no symlink objects in the bucket; + # rewrite them to the real version from the docs_versions dictionary. + # The client URL stays on the alias, and a release only needs a + # dictionary update. + if (req.url ~ "^/ja/(latest|master)/") { + set req.http.X-Docs-Symlink = re.group.1; + set req.http.X-Docs-Version = table.lookup(docs_versions, req.http.X-Docs-Symlink); + if (req.http.X-Docs-Version) { + set req.url = regsub(req.url, "^/ja/(latest|master)/", "/ja/" req.http.X-Docs-Version "/"); + } + } + + # Directory index: a trailing-slash URL fetches index.html from S3 + # (the nginx index directive, as an internal rewrite). + if (req.url ~ "/$") { + set req.url = req.url "index.html"; + } + + # ---- Markdown content negotiation (ja only, where .md twins exist) ---- + # Accept: text/markdown on an .html URL fetches the .md twin instead. + # Pages without a twin (generated indexes, frozen RD versions) fall + # back: vcl_fetch restarts with X-Md-Negotiate=fallback on the 404 and + # this block then restores the .html URL. + if (req.http.X-Md-Negotiate == "fallback") { + if (req.url ~ "\.md$") { + set req.url = regsub(req.url, "\.md$", ".html"); + } + } else if (req.url ~ "^/ja/" && req.url ~ "\.html$" && req.http.Accept == "text/markdown") { + set req.http.X-Md-Negotiate = "md"; + set req.url = regsub(req.url, "\.html$", ".md"); + } + + # ---- backend selection flags, read by the request_conditions ---- + if (req.url ~ "^/capi/en/master/") { + set req.http.X-Docs-Backend = "doxygen"; + set req.url = regsub(req.url, "^/capi/en/master/", "/doxygen-latest-html/"); + } else { + # Narrow this per prefix to move paths over one at a time; anything + # unflagged still goes to docs-origin. + set req.http.X-Docs-Backend = "s3"; + } + } + #FASTLY recv if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") { @@ -15,6 +146,13 @@ sub vcl_fetch { restart; } + # A negotiated .md that does not exist falls back to the .html twin + # (403 is what public-read S3 answers for a missing key). + if ((beresp.status == 403 || beresp.status == 404) && req.http.X-Md-Negotiate == "md" && req.restarts < 3) { + set req.http.X-Md-Negotiate = "fallback"; + restart; + } + if(req.restarts > 0 ) { set beresp.http.Fastly-Restarts = req.restarts; } @@ -36,6 +174,55 @@ sub vcl_fetch { return (deliver); } + if (req.http.X-Docs-Backend) { + # The public-read policy only grants GetObject, so a missing key is + # AccessDenied. Serve it as 404 with a short negative-cache TTL. + if (beresp.status == 403) { + set beresp.status = 404; + set beresp.response = "Not Found"; + } + if (beresp.status == 404) { + set beresp.ttl = 60s; + } + + # Cache-Control and Surrogate-Key move here from the nginx origin; the + # fastly-purge-key key scheme is unchanged. An alias URL (/ja/latest) + # and its real version share one object after the rewrite, so the key + # carries both names: bc-static-all purges ja/ as well as + # ja/latest and ja/master. + if (req.http.X-Orig-Url ~ "^/capi/en/master/") { + set beresp.http.Surrogate-Key = "doxygen-latest-html"; + } else if (req.url ~ "^/(en|ja)/([^/?]+)/") { + set beresp.http.Surrogate-Key = "docs " re.group.1 " " re.group.2 " " re.group.1 "/" re.group.2; + if (req.http.X-Docs-Symlink) { + set beresp.http.Surrogate-Key = beresp.http.Surrogate-Key " ja/" req.http.X-Docs-Symlink; + } + } else { + set beresp.http.Surrogate-Key = "index"; + } + set beresp.http.Cache-Control = "public, max-age=43200, s-maxage=172800, stale-while-revalidate=86400, stale-if-error=604800"; + + # The generated Markdown twins (given text/markdown by nginx since + # ruby/docs.ruby-lang.org#200; aws s3 sync cannot guess a type for .md). + if (req.url ~ "\.md$") { + set beresp.http.Content-Type = "text/markdown; charset=utf-8"; + } + + # The negotiable pages answer differently by Accept, so downstream + # caches need Vary (the edge already keys on the rewritten URL plus + # the normalized Accept). + if (req.http.X-Orig-Url ~ "^/ja/" && req.url ~ "\.(html|md)$") { + set beresp.http.Vary = "Accept"; + } + + unset beresp.http.x-amz-id-2; + unset beresp.http.x-amz-request-id; + unset beresp.http.x-amz-version-id; + unset beresp.http.x-amz-delete-marker; + unset beresp.http.x-amz-server-side-encryption; + unset beresp.http.Server; + } + if (beresp.http.Expires || beresp.http.Surrogate-Control ~ "max-age" || beresp.http.Cache-Control ~"(s-maxage|max-age)") { # keep the ttl here } else { @@ -67,6 +254,31 @@ sub vcl_deliver { sub vcl_error { #FASTLY error + + # Synthetic redirects from vcl_recv: 601 is the permanent kind (the + # nginx return 301 rules), 602 the temporary kind (unreleased-version + # aliases, gone once the docs_versions dictionary moves at a release). + if (obj.status == 601 || obj.status == 602) { + if (obj.status == 601) { + set obj.status = 301; + set obj.response = "Moved Permanently"; + set obj.http.Cache-Control = "public, max-age=3600"; + } else { + set obj.status = 302; + set obj.response = "Found"; + set obj.http.Cache-Control = "public, max-age=60"; + } + set obj.http.Location = req.http.X-Redirect-Location; + synthetic ""; + return(deliver); + } + + # The nginx error_page 50x equivalent. + if (obj.status >= 500 && obj.status < 600) { + set obj.http.Content-Type = "text/html; charset=utf-8"; + synthetic {"Error

An error occurred.

"}; + return(deliver); + } } sub vcl_pass {