From 00e72504bab4c2a7cd35aa18fb0732f208711532 Mon Sep 17 00:00:00 2001
From: Rico <d-Rickyy-b@users.noreply.github.com>
Date: Mon, 20 Apr 2026 01:54:56 +0200
Subject: [PATCH] fix(static-ct): defer partial tile fetching

Optimize tile processing by delaying the fetch of partial tiles. Certstream will wait up to a minute before forcefully fetching the partial tile to ensure slow-growing logs are still processed.

Certstream will now:
- Track when a partial tile is first seen
- Wait up to one minute before forcing a partial fetch
- Skip fetching partial tiles within the deferral window
- Reset tracking when a full tile was downloaded

fixes #104
---
 internal/certificatetransparency/ct-tiled.go | 56 ++++++++++++++++++--
 1 file changed, 51 insertions(+), 5 deletions(-)

diff --git a/internal/certificatetransparency/ct-tiled.go b/internal/certificatetransparency/ct-tiled.go
index bb593d2..1834d50 100644
--- a/internal/certificatetransparency/ct-tiled.go
+++ b/internal/certificatetransparency/ct-tiled.go
@@ -261,12 +261,25 @@ func ConvertTileLeafToRawLogEntry(leaf TileLeaf, index uint64) *ct.RawLogEntry {
 
 	return rawEntry
 }
+
+// DefaultMaxPartialWait is the default maximum time to wait before forcefully
+// fetching a partial tile that has not yet grown into a full tile.
+const DefaultMaxPartialWait = 1 * time.Minute
+
 type StaticCTClient struct {
 	url        string
 	httpClient *http.Client
 	backoff    backoff.Backoff
 	userAgent  string
 	ctIndex    uint64
+
+	// Deferred partial-tile state.
+	// partialTileIndex holds the tile index of the currently tracked partial tile.
+	// partialTileFirstSeen is when that partial tile was first observed; zero means
+	// no partial tile is being tracked.
+	partialTileIndex     uint64
+	partialTileFirstSeen time.Time
+	maxPartialWait       time.Duration
 }
 
 func NewStaticCTClient(url string, httpClient *http.Client, userAgent string, startIndex uint64) *StaticCTClient {
@@ -281,6 +294,7 @@ func NewStaticCTClient(url string, httpClient *http.Client, userAgent string, st
 		},
 		userAgent:      userAgent,
 		ctIndex:        startIndex,
+		maxPartialWait: DefaultMaxPartialWait,
 	}
 }
 
@@ -332,22 +346,54 @@ func (s *StaticCTClient) fetchAndProcessTiles(ctx context.Context, foundCert fun
 	endTile := currentTreeSize / TileSize
 
 	// Process full tiles
+	fetchedFullTiles := false
 	for tileIndex := startTile; tileIndex < endTile; tileIndex++ {
 		if err := s.processTile(ctx, tileIndex, 0, foundCert, foundPrecert); err != nil {
 			return false, fmt.Errorf("processing tile %d: %w", tileIndex, err)
 		}
+
+		fetchedFullTiles = true
+	}
+
+	// When the current end tile has advanced past the tracked partial tile, that tile
+	// has since become a full tile and been processed; reset tracking so we start
+	// fresh for the new partial tile (if any).
+	if endTile > s.partialTileIndex {
+		s.partialTileFirstSeen = time.Time{}
 	}
 
-	// Process partial tile if exists
+	// Process partial tiles.
 	partialSize := currentTreeSize % TileSize
 	if partialSize > 0 {
-		if err := s.processTile(ctx, endTile, partialSize, foundCert, foundPrecert); err != nil {
-			log.Printf("Warning: error processing partial tile %d: %s\n", endTile, err)
-			// Don't return error for partial tiles as they might be incomplete
+		switch {
+		case s.partialTileFirstSeen.IsZero() || s.partialTileIndex != endTile:
+			// First time we see this partial tile – start the deferral clock.
+			s.partialTileIndex = endTile
+			s.partialTileFirstSeen = time.Now()
+			log.Println("Deferring fetch of partial tile", endTile, "with size", partialSize)
+
+		case time.Since(s.partialTileFirstSeen) >= s.maxPartialWait:
+			// The partial tile has been pending too long – fetch it now to prevent
+			// extreme processing delays on slow-growing logs.
+			log.Println("Forcefully fetching partial tile", endTile, "with size", partialSize)
+
+			if err := s.processTile(ctx, endTile, partialSize, foundCert, foundPrecert); err != nil {
+				log.Printf("Warning: error processing partial tile %d: %s\n", endTile, err)
+			}
+
+			// Reset tracking; the tile will be re-observed on the next poll if it
+			// still hasn't grown into a full tile.
+			s.partialTileFirstSeen = time.Time{}
+
+		default:
+			// Still within the deferral window – skip.
 		}
+	} else {
+		// currentTreeSize is an exact multiple of TileSize; no partial tile exists.
+		s.partialTileFirstSeen = time.Time{}
 	}
 
-	return true, nil
+	return fetchedFullTiles, nil
 }
 
 // processTile processes a single tile from the tiled log.