Add google bot IP "firewall"

Author: joecorall

README.md

@@ -157,7 +157,7 @@ To avoid having this middleware impact your SEO score, it's recommended to provi
 A good default value for `goodBots` would be:
 
 ```
-goodBots: apple.com,archive.org,duckduckgo.com,facebook.com,google.com,googlebot.com,googleusercontent.com,instagram.com,kagibot.org,linkedin.com,msn.com,openalex.org,twitter.com,x.com
+goodBots: apple.com,archive.org,duckduckgo.com,facebook.com,google.com,instagram.com,kagibot.org,linkedin.com,msn.com,openalex.org,twitter.com,x.com
 ```
 
 **However** if you set the config parameter `protectParameters="true"`, even good bots won't be allowed to crawl protected routes if a URL parameter is on the request (e.g. `/foo?bar=baz`). This `protectParameters` feature is meant to help protect faceted search pages.

ci/docker-compose.yml

@@ -19,6 +19,7 @@ services:
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.ipForwardedHeader: "X-Forwarded-For"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.logLevel: "DEBUG"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: ""
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableGooglebotIPCheck: "true"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.protectRoutes: "/"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.persistentStateFile: "/tmp/state.json"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStateReconciliation: "true"
@@ -48,6 +49,7 @@ services:
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.ipForwardedHeader: "X-Forwarded-For"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.logLevel: "DEBUG"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: ""
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableGooglebotIPCheck: "true"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.protectRoutes: "/"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.persistentStateFile: "/tmp/state.json"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStateReconciliation: "true"

internal/state/google.go

@@ -0,0 +1,53 @@
+package state
+
+import (
+	"log"
+	"net"
+	"sync"
+)
+
+// GooglebotIPs holds the list of Googlebot IP ranges.
+type GooglebotIPs struct {
+	cidrs []*net.IPNet
+	mu    sync.RWMutex
+}
+
+// NewGooglebotIPs creates a new GooglebotIPs struct.
+func NewGooglebotIPs() *GooglebotIPs {
+	return &GooglebotIPs{
+		cidrs: make([]*net.IPNet, 0),
+	}
+}
+
+// Update replaces the current list of CIDRs with a new one.
+func (g *GooglebotIPs) Update(cidrs []string) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	g.cidrs = make([]*net.IPNet, 0, len(cidrs))
+
+	for _, s := range cidrs {
+		_, network, err := net.ParseCIDR(s)
+		if err != nil {
+			log.Printf("error parsing CIDR %s: %v", s, err)
+
+			continue
+		}
+
+		g.cidrs = append(g.cidrs, network)
+	}
+}
+
+// Contains checks if a given IP is in the list of Googlebot IPs.
+func (g *GooglebotIPs) Contains(ip net.IP) bool {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
+	for _, network := range g.cidrs {
+		if network.Contains(ip) {
+			return true
+		}
+	}
+
+	return false
+}

main.go

@@ -75,6 +75,7 @@ type Config struct {
 	// Only enable this if running multiple plugin instances sharing the same state file.
 	// Performance warning: Not recommended for sites with >1M unique visitors (see internal/state/state_stress_test.go).
 	EnableStateReconciliation string `json:"enableStateReconciliation"`
+	EnableGooglebotIPCheck    string `json:"enableGooglebotIPCheck"`
 	Mode                      string `json:"mode"`
 	PeriodSeconds             int    `json:"periodSeconds"`
 	FailureThreshold          int    `json:"failureThreshold"`
@@ -89,6 +90,7 @@ type CaptchaProtect struct {
 	rateCache          *lru.Cache
 	verifiedCache      *lru.Cache
 	botCache           *lru.Cache
+	googlebotIPs       *state.GooglebotIPs
 	captchaConfig      CaptchaConfig
 	exemptIps          []*net.IPNet
 	tmpl               *template.Template
@@ -138,6 +140,7 @@ func CreateConfig() *Config {
 		CaptchaProvider:           "turnstile",
 		Mode:                      "prefix",
 		EnableStateReconciliation: "false",
+		EnableGooglebotIPCheck:    "false",
 		PeriodSeconds:             DefaultHealthCheckPeriodSeconds,
 		FailureThreshold:          DefaultHealthCheckFailureThreshold,
 	}
@@ -329,9 +332,91 @@ func NewCaptchaProtect(ctx context.Context, next http.Handler, config *Config, n
 		}()
 	}
 
+	if config.EnableGooglebotIPCheck == "true" {
+		log.Info("Googlebot IP check enabled")
+		bc.googlebotIPs = state.NewGooglebotIPs()
+		childCtx, cancel := context.WithCancel(ctx)
+		go bc.googlebotIPCheckLoop(childCtx)
+		go func() {
+			<-ctx.Done()
+			log.Debug("Context canceled, stopping Googlebot IP check loop")
+			cancel()
+		}()
+	}
+
 	return &bc, nil
 }
 
+type googlebotIPs struct {
+	Prefixes []struct {
+		IPv4Prefix string `json:"ipv4Prefix"`
+		IPv6Prefix string `json:"ipv6Prefix"`
+	} `json:"prefixes"`
+}
+
+func (bc *CaptchaProtect) googlebotIPCheckLoop(ctx context.Context) {
+	ticker := time.NewTicker(24 * time.Hour)
+	defer ticker.Stop()
+
+	// Initial fetch
+	bc.fetchGooglebotIPs()
+
+	for {
+		select {
+		case <-ticker.C:
+			bc.fetchGooglebotIPs()
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (bc *CaptchaProtect) fetchGooglebotIPs() {
+	bc.log.Debug("Fetching Googlebot IPs")
+
+	req, err := http.NewRequest(http.MethodGet, "https://developers.google.com/static/search/apis/ipranges/googlebot.json", nil)
+	if err != nil {
+		bc.log.Error("Failed to create Googlebot IP request", "err", err)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	req = req.WithContext(ctx)
+
+	resp, err := bc.httpClient.Do(req)
+	if err != nil {
+		bc.log.Error("Failed to fetch Googlebot IPs", "err", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		bc.log.Error("Failed to fetch Googlebot IPs", "statusCode", resp.StatusCode)
+		return
+	}
+
+	var ips googlebotIPs
+	err = json.NewDecoder(resp.Body).Decode(&ips)
+	if err != nil {
+		bc.log.Error("Failed to decode Googlebot IPs", "err", err)
+		return
+	}
+
+	var cidrs []string
+	for _, prefix := range ips.Prefixes {
+		if prefix.IPv4Prefix != "" {
+			cidrs = append(cidrs, prefix.IPv4Prefix)
+		}
+		if prefix.IPv6Prefix != "" {
+			cidrs = append(cidrs, prefix.IPv6Prefix)
+		}
+	}
+
+	bc.googlebotIPs.Update(cidrs)
+	bc.log.Info("Updated Googlebot IPs", "count", len(cidrs))
+}
+
 // getCaptchaConfig returns the captcha configuration for a given provider.
 // Returns an empty CaptchaConfig if the provider is invalid.
 func getCaptchaConfig(provider string) CaptchaConfig {
@@ -899,6 +984,12 @@ func (bc *CaptchaProtect) isGoodBot(req *http.Request, clientIP string) bool {
 		}
 	}
 
+	if bc.config.EnableGooglebotIPCheck == "true" {
+		if bc.googlebotIPs.Contains(net.ParseIP(clientIP)) {
+			return true
+		}
+	}
+
 	bot, ok := bc.botCache.Get(clientIP)
 	if ok {
 		return bot.(bool)