Skip to content

Commit 982c26b

Browse files
authored
cache: redesign as a Bazel-style CAS and action cache (#4539)
1 parent 98ef75d commit 982c26b

8 files changed

Lines changed: 713 additions & 74 deletions

File tree

docs/reference/environment-variables.md

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,26 @@ they are introduced.
2020
## SQLCCACHE
2121

2222
The `SQLCCACHE` environment variable dictates where `sqlc` will store cached
23-
WASM-based plugins and modules. By default `sqlc` follows the [XDG Base
24-
Directory
23+
data. By default `sqlc` follows the [XDG Base Directory
2524
Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
2625

26+
The cache is designed after Bazel's local disk cache and has three parts:
27+
28+
- `cas/` — a content-addressable store holding blobs (query analysis
29+
results, WASM plugin binaries, compiled WASM machine code) keyed by the
30+
SHA-256 hash of their contents. A remotely fetched plugin's address is
31+
exactly the checksum declared in the configuration file, so it is loaded
32+
directly by that address.
33+
- `ac/` — an action cache mapping the digest of a unit of cacheable work and
34+
its inputs (analyzing a query against a schema, compiling a WASM module to
35+
machine code) to the CAS digests of its outputs.
36+
- `exec/` — per-action directories where cached output trees are
37+
materialized for tools that read them from disk, such as the
38+
[wazero](https://wazero.io) runtime's compilation cache.
39+
40+
The entire directory is safe to delete at any time; sqlc will rebuild it as
41+
needed.
42+
2743
## SQLCDEBUG
2844

2945
The `SQLCDEBUG` variable controls debugging variables within the runtime. It is

internal/analyzer/analyzer.go

Lines changed: 42 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,14 @@ package analyzer
33
import (
44
"context"
55
"encoding/json"
6-
"fmt"
7-
"hash/fnv"
6+
"errors"
87
"log/slog"
9-
"os"
10-
"path/filepath"
118

129
"google.golang.org/protobuf/proto"
1310

1411
"github.com/sqlc-dev/sqlc/internal/analysis"
1512
"github.com/sqlc-dev/sqlc/internal/cache"
1613
"github.com/sqlc-dev/sqlc/internal/config"
17-
"github.com/sqlc-dev/sqlc/internal/info"
1814
"github.com/sqlc-dev/sqlc/internal/sql/ast"
1915
"github.com/sqlc-dev/sqlc/internal/sql/named"
2016
)
@@ -24,6 +20,7 @@ type CachedAnalyzer struct {
2420
config config.Config
2521
configBytes []byte
2622
db config.Database
23+
store *cache.Cache
2724
}
2825

2926
func Cached(a Analyzer, c config.Config, db config.Database) *CachedAnalyzer {
@@ -47,46 +44,54 @@ func (c *CachedAnalyzer) Analyze(ctx context.Context, n ast.Node, q string, sche
4744
return result, err
4845
}
4946

47+
// The name of the sole output blob a QueryAnalysis action produces.
48+
const analysisOutput = "analysis.pb"
49+
5050
func (c *CachedAnalyzer) analyze(ctx context.Context, n ast.Node, q string, schema []string, np *named.ParamSet) (*analysis.Analysis, bool, error) {
5151
// Only cache queries for managed databases. We can't be certain the
5252
// database is in an unchanged state otherwise
5353
if !c.db.Managed {
5454
return nil, true, nil
5555
}
5656

57-
dir, err := cache.AnalysisDir()
58-
if err != nil {
59-
return nil, true, err
57+
if c.store == nil {
58+
var err error
59+
c.store, err = cache.Open()
60+
if err != nil {
61+
return nil, true, err
62+
}
6063
}
64+
store := c.store
6165

6266
if c.configBytes == nil {
67+
var err error
6368
c.configBytes, err = json.Marshal(c.config)
6469
if err != nil {
6570
return nil, true, err
6671
}
6772
}
6873

69-
// Calculate cache key
70-
h := fnv.New64()
71-
h.Write([]byte(info.Version))
72-
h.Write(c.configBytes)
74+
// Analyzing a query is an action whose inputs are the configuration, the
75+
// schema migrations, and the query itself. (The sqlc binary is an
76+
// implicit input of every action.)
77+
action := store.NewAction("QueryAnalysis").
78+
AddInput("config", c.configBytes)
7379
for _, m := range schema {
74-
h.Write([]byte(m))
80+
action.AddInput("schema", []byte(m))
7581
}
76-
h.Write([]byte(q))
77-
78-
key := fmt.Sprintf("%x", h.Sum(nil))
79-
path := filepath.Join(dir, key)
80-
if _, err := os.Stat(path); err == nil {
81-
contents, err := os.ReadFile(path)
82-
if err != nil {
83-
return nil, true, err
82+
actionDigest := action.AddInput("query", []byte(q)).Digest()
83+
84+
if cached, err := store.Actions.Get(actionDigest); err == nil {
85+
contents, err := store.CAS.Get(cached.Outputs[analysisOutput])
86+
if err == nil {
87+
var a analysis.Analysis
88+
if err := proto.Unmarshal(contents, &a); err == nil {
89+
return &a, false, nil
90+
}
8491
}
85-
var a analysis.Analysis
86-
if err := proto.Unmarshal(contents, &a); err != nil {
87-
return nil, true, err
92+
if !errors.Is(err, cache.ErrNotFound) {
93+
slog.Warn("reading analysis from cache failed", "err", err)
8894
}
89-
return &a, false, nil
9095
}
9196

9297
result, err := c.a.Analyze(ctx, n, q, schema, np)
@@ -97,16 +102,26 @@ func (c *CachedAnalyzer) analyze(ctx context.Context, n ast.Node, q string, sche
97102
slog.Warn("unable to marshal analysis", "err", err)
98103
return result, false, nil
99104
}
100-
if err := os.WriteFile(path, contents, 0644); err != nil {
101-
slog.Warn("saving analysis to disk failed", "err", err)
105+
outDigest, err := store.CAS.Put(contents)
106+
if err != nil {
107+
slog.Warn("saving analysis to cache failed", "err", err)
102108
return result, false, nil
103109
}
110+
err = store.Actions.Put(actionDigest, &cache.ActionResult{
111+
Outputs: map[string]cache.Digest{analysisOutput: outDigest},
112+
})
113+
if err != nil {
114+
slog.Warn("saving analysis action result failed", "err", err)
115+
}
104116
}
105117

106118
return result, false, err
107119
}
108120

109121
func (c *CachedAnalyzer) Close(ctx context.Context) error {
122+
if c.store != nil {
123+
c.store.Close()
124+
}
110125
return c.a.Close(ctx)
111126
}
112127

internal/cache/action.go

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
package cache
2+
3+
import (
4+
"encoding/json"
5+
"errors"
6+
"fmt"
7+
"io/fs"
8+
"os"
9+
"path/filepath"
10+
)
11+
12+
// ActionResult records the outputs of a completed action, mirroring Bazel's
13+
// ActionResult message. Outputs are not stored inline: each named output is a
14+
// digest pointing into the CAS.
15+
type ActionResult struct {
16+
// Outputs maps an output name (e.g. "analysis.pb", "plugin.wasm") to the
17+
// CAS digest of its contents.
18+
Outputs map[string]Digest `json:"outputs"`
19+
}
20+
21+
// ActionCache maps action digests to ActionResults, stored as JSON files at
22+
// ac/<xx>/<hash> under the cache root. Unlike CAS entries, action cache
23+
// entries are not self-validating — the value is not derivable from the key —
24+
// so Get additionally checks that every referenced output still exists in the
25+
// CAS before reporting a hit, exactly like Bazel's disk cache does.
26+
//
27+
// Entry I/O goes through the same os.Root as the CAS, confining every read
28+
// and write to the cache directory.
29+
type ActionCache struct {
30+
root *os.Root
31+
cas *CAS
32+
}
33+
34+
func newActionCache(root *os.Root, cas *CAS) *ActionCache {
35+
return &ActionCache{root: root, cas: cas}
36+
}
37+
38+
// path returns an entry's path relative to the cache root.
39+
func (a *ActionCache) path(d Digest) string {
40+
return filepath.Join("ac", d.Hash[:2], d.Hash)
41+
}
42+
43+
// Get returns the cached result for an action, or ErrNotFound on a miss. An
44+
// entry whose outputs are missing or corrupt in the CAS is treated as a miss
45+
// and evicted.
46+
func (a *ActionCache) Get(action Digest) (*ActionResult, error) {
47+
if !action.valid() {
48+
return nil, ErrNotFound
49+
}
50+
path := a.path(action)
51+
data, err := a.root.ReadFile(path)
52+
if err != nil {
53+
if errors.Is(err, fs.ErrNotExist) {
54+
return nil, ErrNotFound
55+
}
56+
return nil, fmt.Errorf("cache: %w", err)
57+
}
58+
var result ActionResult
59+
if err := json.Unmarshal(data, &result); err != nil {
60+
a.root.Remove(path)
61+
return nil, ErrNotFound
62+
}
63+
for _, d := range result.Outputs {
64+
if !a.cas.Contains(d) {
65+
a.root.Remove(path)
66+
return nil, ErrNotFound
67+
}
68+
}
69+
return &result, nil
70+
}
71+
72+
// PutTree stores every file under dir in the CAS and records them as the
73+
// action's outputs, named by their paths relative to dir. Use this for
74+
// actions whose tool writes an output directory, like WASM compilation.
75+
func (a *ActionCache) PutTree(action Digest, dir string) error {
76+
outputs := map[string]Digest{}
77+
err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error {
78+
if err != nil || entry.IsDir() {
79+
return err
80+
}
81+
rel, err := filepath.Rel(dir, path)
82+
if err != nil {
83+
return err
84+
}
85+
data, err := os.ReadFile(path)
86+
if err != nil {
87+
return err
88+
}
89+
d, err := a.cas.Put(data)
90+
if err != nil {
91+
return err
92+
}
93+
outputs[filepath.ToSlash(rel)] = d
94+
return nil
95+
})
96+
if err != nil {
97+
return fmt.Errorf("cache: %w", err)
98+
}
99+
if len(outputs) == 0 {
100+
return fmt.Errorf("cache: no outputs found under %s", dir)
101+
}
102+
return a.Put(action, &ActionResult{Outputs: outputs})
103+
}
104+
105+
// GetTree materializes a cached action's outputs as files under dir, or
106+
// returns ErrNotFound on a miss. A file already present is reused only if its
107+
// contents still hash to the expected digest; otherwise it is rewritten from
108+
// the CAS. Writes are staged, fsynced, and renamed, so a crash cannot leave a
109+
// right-sized but torn file that later reads would trust.
110+
func (a *ActionCache) GetTree(action Digest, dir string) error {
111+
result, err := a.Get(action)
112+
if err != nil {
113+
return err
114+
}
115+
for rel, d := range result.Outputs {
116+
// Output names come from the action cache entry, which — unlike a CAS
117+
// blob — is not self-validating, so a tampered entry could carry a
118+
// name like "../../etc/x". Reject anything that isn't a relative path
119+
// confined to dir; this is the confinement the os.Root gives the rest
120+
// of the cache, which GetTree can't use because dir is a caller-owned
121+
// path a tool must read by absolute name.
122+
if !filepath.IsLocal(rel) {
123+
return fmt.Errorf("cache: unsafe output path %q in action %s", rel, action)
124+
}
125+
path := filepath.Join(dir, filepath.FromSlash(rel))
126+
// Trust an existing file only if it still hashes to the digest;
127+
// size alone can mask in-place corruption that would make the
128+
// consuming tool hard-fail with no way to repair (see wazero).
129+
if existing, err := os.ReadFile(path); err == nil && DigestOf(existing) == d {
130+
continue
131+
}
132+
data, err := a.cas.Get(d)
133+
if err != nil {
134+
return err
135+
}
136+
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
137+
return fmt.Errorf("cache: %w", err)
138+
}
139+
if err := writeFileAtomic(path, data); err != nil {
140+
return fmt.Errorf("cache: %w", err)
141+
}
142+
}
143+
return nil
144+
}
145+
146+
// writeFileAtomic writes data to path via a staged temp file in the same
147+
// directory, fsynced before an atomic rename, so a reader never observes a
148+
// partial or torn file even across a crash.
149+
func writeFileAtomic(path string, data []byte) error {
150+
f, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+"-*")
151+
if err != nil {
152+
return err
153+
}
154+
tmp := f.Name()
155+
if _, err := f.Write(data); err != nil {
156+
f.Close()
157+
os.Remove(tmp)
158+
return err
159+
}
160+
if err := f.Sync(); err != nil {
161+
f.Close()
162+
os.Remove(tmp)
163+
return err
164+
}
165+
if err := f.Close(); err != nil {
166+
os.Remove(tmp)
167+
return err
168+
}
169+
if err := os.Rename(tmp, path); err != nil {
170+
os.Remove(tmp)
171+
return err
172+
}
173+
return nil
174+
}
175+
176+
// Put records the result of an action. All outputs must already be in the
177+
// CAS; writes are staged and renamed so concurrent processes never observe a
178+
// partial entry.
179+
func (a *ActionCache) Put(action Digest, result *ActionResult) error {
180+
for name, d := range result.Outputs {
181+
if !a.cas.Contains(d) {
182+
return fmt.Errorf("cache: output %q (%s) missing from CAS", name, d)
183+
}
184+
}
185+
data, err := json.Marshal(result)
186+
if err != nil {
187+
return fmt.Errorf("cache: %w", err)
188+
}
189+
path := a.path(action)
190+
if err := a.root.MkdirAll(filepath.Dir(path), 0755); err != nil {
191+
return fmt.Errorf("cache: %w", err)
192+
}
193+
f, name, err := a.cas.createTemp(action.Hash[:8] + "-")
194+
if err != nil {
195+
return fmt.Errorf("cache: %w", err)
196+
}
197+
defer a.root.Remove(name)
198+
if _, err := f.Write(data); err != nil {
199+
f.Close()
200+
return fmt.Errorf("cache: %w", err)
201+
}
202+
if err := f.Close(); err != nil {
203+
return fmt.Errorf("cache: %w", err)
204+
}
205+
if err := a.root.Rename(name, path); err != nil {
206+
return fmt.Errorf("cache: %w", err)
207+
}
208+
return nil
209+
}

0 commit comments

Comments
 (0)