core: cache the seeded catalog in the action cache

Seeding a dialect writes thousands of rows — PostgreSQL's functions and
system catalogs alone run to five figures — and produces the same database
every time. So the seeded SQLite database is now serialized into the CAS
and deserialized back on the next run, keyed by an action naming the
dialect.

The dialect data that determines the catalog is embedded in the sqlc
binary, and the binary's digest is already an input to every action, so
nothing else has to be hashed: a rebuilt sqlc misses, an unchanged one
hits. A cache that cannot be opened, read or written is not an error — a
corrupt blob, a missing one and a read-only cache directory all fall back
to seeding.

Per analyze run, catalog setup included: PostgreSQL 112ms -> 29ms, MySQL
31ms -> 17ms, SQLite 27ms -> 17ms. The endtoend suite runs in half the
time on a warm cache.

Co-Authored-By: Claude <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011MnoUabwBWW9gaEn2Nj7eG

Author: claude

internal/compiler/engine.go

@@ -178,7 +178,9 @@ func (c *Compiler) initCore() error {
 	default:
 		return fmt.Errorf("unknown engine: %s", c.conf.Engine)
 	}
-	cat, err := core.New(dialect)
+	// The seeded catalog is the same on every run, so it is cached rather
+	// than rebuilt. The engine names which dialect is being asked for.
+	cat, err := core.NewCached(string(c.conf.Engine), dialect)
 	if err != nil {
 		return fmt.Errorf("%s: init catalog: %w", c.conf.Engine, err)
 	}

internal/core/cache.go

@@ -0,0 +1,151 @@
+package core
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"log/slog"
+
+	"github.com/sqlc-dev/sqlc/internal/cache"
+	"github.com/sqlc-dev/sqlc/internal/core/catalogdb"
+)
+
+// The name of the sole output blob a CoreCatalog action produces.
+const catalogOutput = "catalog.db"
+
+// serializer is the driver's serialization API, which the SQLite driver
+// implements on its connections.
+type serializer interface {
+	Serialize() ([]byte, error)
+	Deserialize([]byte) error
+}
+
+// NewCached returns a catalog seeded by opts, restored from the on-disk cache
+// when one seeded from the same inputs has been built before.
+//
+// Seeding a dialect means writing thousands of rows — PostgreSQL's functions
+// and system catalogs alone run to five figures — and the result is the same
+// on every run. So the seeded database is serialized into the cache and
+// deserialized back on the next one, which turns the work into a single read.
+//
+// The catalog a dialect seeds is determined entirely by the dialect data, and
+// that data is embedded in the sqlc binary, whose digest is an input to every
+// action. So the action needs only enough to tell one dialect from another.
+//
+// A cache that cannot be opened, read or written is not an error: the catalog
+// is simply built the long way.
+func NewCached(dialect string, opts ...Option) (*Catalog, error) {
+	store, err := cache.Open()
+	if err != nil {
+		slog.Debug("opening the cache failed", "err", err)
+		return New(opts...)
+	}
+	defer store.Close()
+
+	action := store.NewAction("CoreCatalog").AddInput("dialect", []byte(dialect)).Digest()
+	if c, err := restore(store, action); err == nil {
+		return c, nil
+	} else if !errors.Is(err, cache.ErrNotFound) {
+		slog.Debug("restoring the catalog from the cache failed", "err", err)
+	}
+
+	c, err := New(opts...)
+	if err != nil {
+		return nil, err
+	}
+	if err := save(store, action, c); err != nil {
+		slog.Debug("saving the catalog to the cache failed", "err", err)
+	}
+	return c, nil
+}
+
+// restore opens a catalog from the database a previous run cached.
+func restore(store *cache.Cache, action cache.Digest) (*Catalog, error) {
+	result, err := store.Actions.Get(action)
+	if err != nil {
+		return nil, err
+	}
+	blob, err := store.CAS.Get(result.Outputs[catalogOutput])
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := openDB()
+	if err != nil {
+		return nil, err
+	}
+	if err := deserialize(db, blob); err != nil {
+		db.Close()
+		return nil, err
+	}
+
+	stmts := newStmtCache(db)
+	c := &Catalog{db: db, stmts: stmts, q: catalogdb.New(stmts)}
+
+	// The seeds recorded the dialect on the catalog as they ran. A restored
+	// catalog reads it back from the database instead.
+	row, err := c.q.SeededDialect(context.Background())
+	if err != nil {
+		c.Close()
+		return nil, fmt.Errorf("core: restored catalog has no dialect: %w", err)
+	}
+	c.dialectOID = row.Oid
+	return c, nil
+}
+
+// save serializes a freshly seeded catalog into the cache.
+func save(store *cache.Cache, action cache.Digest, c *Catalog) error {
+	blob, err := serialize(c.db)
+	if err != nil {
+		return err
+	}
+	digest, err := store.CAS.Put(blob)
+	if err != nil {
+		return err
+	}
+	return store.Actions.Put(action, &cache.ActionResult{
+		Outputs: map[string]cache.Digest{catalogOutput: digest},
+	})
+}
+
+// serialize returns the bytes a database would be written to disk as.
+func serialize(db *sql.DB) ([]byte, error) {
+	var blob []byte
+	err := withRawConn(db, func(s serializer) error {
+		var err error
+		blob, err = s.Serialize()
+		return err
+	})
+	if err != nil {
+		return nil, fmt.Errorf("core: serialize catalog: %w", err)
+	}
+	return blob, nil
+}
+
+// deserialize loads a serialized database into an open one. The pool is
+// pinned to a single connection, so the connection this replaces the contents
+// of is the only one the catalog will ever use.
+func deserialize(db *sql.DB, blob []byte) error {
+	if err := withRawConn(db, func(s serializer) error {
+		return s.Deserialize(blob)
+	}); err != nil {
+		return fmt.Errorf("core: deserialize catalog: %w", err)
+	}
+	return nil
+}
+
+func withRawConn(db *sql.DB, fn func(serializer) error) error {
+	conn, err := db.Conn(context.Background())
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	return conn.Raw(func(driverConn any) error {
+		s, ok := driverConn.(serializer)
+		if !ok {
+			return fmt.Errorf("driver does not serialize databases")
+		}
+		return fn(s)
+	})
+}

internal/core/catalog.go

@@ -30,29 +30,9 @@ func WithSeed(fn func(*Catalog) error) Option {
 }
 
 func New(opts ...Option) (*Catalog, error) {
-	db, err := sql.Open("sqlite", ":memory:")
+	db, err := openDB()
 	if err != nil {
-		return nil, fmt.Errorf("core: open catalog: %w", err)
-	}
-	// Every ":memory:" connection is its own empty database, so the pool has to
-	// be pinned to a single connection that is never retired — otherwise a
-	// second connection would see a catalog with no tables in it.
-	db.SetMaxOpenConns(1)
-	db.SetMaxIdleConns(1)
-	db.SetConnMaxIdleTime(0)
-	db.SetConnMaxLifetime(0)
-
-	// The catalog is scratch state rebuilt on every run and never read back
-	// from disk, so durability buys nothing and costs a journal write per
-	// statement.
-	for _, pragma := range []string{
-		"PRAGMA journal_mode = OFF",
-		"PRAGMA synchronous = OFF",
-	} {
-		if _, err := db.Exec(pragma); err != nil {
-			db.Close()
-			return nil, fmt.Errorf("core: %s: %w", pragma, err)
-		}
+		return nil, err
 	}
 
 	// catalogdef.Schema is a batch of DDL statements, so it goes to the
@@ -84,6 +64,35 @@ func New(opts ...Option) (*Catalog, error) {
 	return c, nil
 }
 
+// openDB opens the empty SQLite database a catalog is kept in.
+func openDB() (*sql.DB, error) {
+	db, err := sql.Open("sqlite", ":memory:")
+	if err != nil {
+		return nil, fmt.Errorf("core: open catalog: %w", err)
+	}
+	// Every ":memory:" connection is its own empty database, so the pool has to
+	// be pinned to a single connection that is never retired — otherwise a
+	// second connection would see a catalog with no tables in it.
+	db.SetMaxOpenConns(1)
+	db.SetMaxIdleConns(1)
+	db.SetConnMaxIdleTime(0)
+	db.SetConnMaxLifetime(0)
+
+	// The catalog is scratch state rebuilt on every run and never read back
+	// from disk, so durability buys nothing and costs a journal write per
+	// statement.
+	for _, pragma := range []string{
+		"PRAGMA journal_mode = OFF",
+		"PRAGMA synchronous = OFF",
+	} {
+		if _, err := db.Exec(pragma); err != nil {
+			db.Close()
+			return nil, fmt.Errorf("core: %s: %w", pragma, err)
+		}
+	}
+	return db, nil
+}
+
 func (c *Catalog) setup(opts []Option) error {
 	if err := c.bootstrap(); err != nil {
 		return fmt.Errorf("core: bootstrap: %w", err)

internal/core/catalogdb/query.sql.go

@@ -20,6 +20,11 @@ INSERT INTO sql_dialect (name) VALUES (?);
 -- name: DialectOID :one
 SELECT oid FROM sql_dialect WHERE name = ?;
 
+-- name: SeededDialect :one
+-- The dialect a catalog was seeded with. A catalog is built for exactly one,
+-- so a restored catalog finds it without being told its name.
+SELECT oid, name FROM sql_dialect ORDER BY oid LIMIT 1;
+
 -- name: SetDialectFlag :exec
 INSERT INTO sql_dialect_flag (dialect_oid, key, value)
 VALUES (?, ?, ?)