fix(migrations): robustly strip psql meta commands without breaking SQL

Replace naive PostgreSQL schema preprocessing with a single-pass state machine that distinguishes top-level psql meta-commands from valid SQL backslashes, literals, identifiers, comments, and dollar-quoted bodies.

The previous implementation could leave pg_dump/client backslash directives in schema-loading paths or strip too aggressively, breaking valid SQL containing:
- Backslashes in string literals, including `E'...'` escapes and simple `standard_conforming_strings` variants
- Meta-command text in comments or documentation
- Dollar-quoted function bodies, including Unicode-tagged bodies
- Double-quoted identifiers and identifiers containing `$`

Changes:
- Add engine-aware `PreprocessSchema()` and `PreprocessSchemaForApply()` helpers so rollback removal always applies while PostgreSQL psql stripping is mode-aware.
- Replace line-based PostgreSQL filtering with a single-pass lexer that tracks single quotes, double quotes, dollar quotes, line comments, nested block comments, and statement boundaries.
- Handle escape-string prefixes, simple `standard_conforming_strings` changes, Unicode dollar-quote tags, identifier-boundary checks, documented psql meta-commands, and broader unknown top-level backslash directives.
- Preserve SQL after a valid inline `\\` separator that follows a meta-command, including glued and one-sided-whitespace forms observed in psql 13.22 / 14.19 / 15.14 / 16.10 / 17.6 / 17.10; preserve invalid leading `\\` input instead of normalizing it into SQL.
- Strip semantic psql commands such as `\connect`, includes, `\copy`, `\gexec`, `\q`, `\quit`, and `\r` with warnings in parse/codegen paths, but reject them in schema-application paths where sqlc cannot reproduce their effects safely.
- Reject psql conditionals (`\if`, `\elif`, `\else`, `\endif`) instead of flattening branches and changing SQL semantics.
- Remove `\copy ... from stdin` payload rows through an exact `\.` terminator in parse mode, and reject unterminated copy data.
- Treat `standard_conforming_strings` and transaction-scoped script behavior as best-effort parsing aids rather than full psql emulation; report approximation warnings in parse mode while suppressing that parse-only warning for live apply mode.
- Wire preprocessing and warning propagation into compiler parsing, generate processing, `createdb`, `verify`, managed `vet`, and PostgreSQL sqltest seeding paths.
- Add regression coverage for documented meta-commands, unknown directives, literals, comments, dollar quotes, inline separators, semantic warnings, apply-mode rejections, copy data, line endings, and managed/PostgreSQL preprocessing rollout.

Performance improvements:
- Pre-allocate output buffers with `strings.Builder.Grow()`.
- Keep parsing single-pass rather than rescanning line slices.
- Reuse engine-aware preprocessing helpers across schema-loading paths.

Testing:
- `go test ./internal/migrations ./internal/compiler ./internal/schemautil ./internal/cmd ./internal/sqltest/...`

Author: ignatremizov

internal/cmd/createdb.go

@@ -84,7 +84,14 @@ func CreateDB(ctx context.Context, dir, filename, querySetName string, o *Option
 		if err != nil {
 			return fmt.Errorf("read file: %w", err)
 		}
-		ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
+		ddlText, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(queryset.Engine))
+		if err != nil {
+			return err
+		}
+		for _, warning := range warnings {
+			fmt.Fprintln(o.Stderr, warning)
+		}
+		ddl = append(ddl, ddlText)
 	}
 
 	now := time.Now().UTC().UnixNano()

internal/cmd/generate.go

@@ -265,6 +265,9 @@ func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.C
 		return nil, true
 	}
 	if err := c.ParseCatalog(sql.Schema); err != nil {
+		for _, warning := range c.Warnings() {
+			fmt.Fprintln(stderr, warning)
+		}
 		fmt.Fprintf(stderr, "# package %s\n", name)
 		if parserErr, ok := err.(*multierr.Error); ok {
 			for _, fileErr := range parserErr.Errs() {
@@ -275,6 +278,9 @@ func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.C
 		}
 		return nil, true
 	}
+	for _, warning := range c.Warnings() {
+		fmt.Fprintln(stderr, warning)
+	}
 	if debugDumpCatalog.Value() == "1" {
 		debug.Dump(c.Catalog())
 	}

internal/cmd/process.go

@@ -119,12 +119,12 @@ func processQuerySets(ctx context.Context, rp ResultProcessor, conf *config.Conf
 	if err := grp.Wait(); err != nil {
 		return err
 	}
-	if errored {
-		for i, _ := range stderrs {
-			if _, err := io.Copy(stderr, &stderrs[i]); err != nil {
-				return err
-			}
+	for i := range stderrs {
+		if _, err := io.Copy(stderr, &stderrs[i]); err != nil {
+			return err
 		}
+	}
+	if errored {
 		return fmt.Errorf("errored")
 	}
 	return nil

internal/cmd/verify.go

@@ -102,7 +102,14 @@ func Verify(ctx context.Context, dir, filename string, opts *Options) error {
 				if err != nil {
 					return fmt.Errorf("read file: %w", err)
 				}
-				ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
+				ddlText, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(current.Engine))
+				if err != nil {
+					return err
+				}
+				for _, warning := range warnings {
+					fmt.Fprintln(stderr, warning)
+				}
+				ddl = append(ddl, ddlText)
 			}
 
 			var codegen plugin.GenerateRequest

internal/cmd/vet.go

@@ -439,7 +439,14 @@ func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, f
 		if err != nil {
 			return "", cleanup, fmt.Errorf("read file: %w", err)
 		}
-		ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
+		ddlText, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(s.Engine))
+		if err != nil {
+			return "", cleanup, err
+		}
+		for _, warning := range warnings {
+			fmt.Fprintln(c.Stderr, warning)
+		}
+		ddl = append(ddl, ddlText)
 	}
 
 	resp, err := c.Client.CreateDatabase(ctx, &dbmanager.CreateDatabaseRequest{
@@ -554,7 +561,13 @@ func (c *checker) checkSQL(ctx context.Context, s config.SQL) error {
 					if err != nil {
 						return fmt.Errorf("read schema file: %w", err)
 					}
-					ddl := migrations.RemoveRollbackStatements(string(contents))
+					ddl, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(s.Engine))
+					if err != nil {
+						return err
+					}
+					for _, warning := range warnings {
+						fmt.Fprintln(c.Stderr, warning)
+					}
 					if _, err := db.ExecContext(ctx, ddl); err != nil {
 						return fmt.Errorf("apply schema %s: %w", schema, err)
 					}

internal/compiler/compile.go

@@ -38,9 +38,24 @@ func (c *Compiler) parseCatalog(schemas []string) error {
 			merr.Add(filename, "", 0, err)
 			continue
 		}
-		contents := migrations.RemoveRollbackStatements(string(blob))
-		contents = migrations.RemovePsqlMetaCommands(contents)
+		contents, warnings, err := migrations.PreprocessSchema(string(blob), string(c.conf.Engine))
+		if err != nil {
+			merr.Add(filename, string(blob), 0, err)
+			continue
+		}
+		var applyContents string
+		if c.usesManagedAnalyzer() {
+			applyContents, _, err = migrations.PreprocessSchemaForApply(string(blob), string(c.conf.Engine))
+			if err != nil {
+				merr.Add(filename, string(blob), 0, err)
+				continue
+			}
+		}
+		c.warns = append(c.warns, warnings...)
 		c.schema = append(c.schema, contents)
+		if c.usesManagedAnalyzer() {
+			c.applySchema = append(c.applySchema, applyContents)
+		}
 
 		// In database-only mode, we parse the schema to validate syntax
 		// but don't update the catalog - the database will be the source of truth
@@ -73,7 +88,7 @@ func (c *Compiler) parseQueries(o opts.Parser) (*Result, error) {
 
 	// In database-only mode, initialize the database connection before parsing queries
 	if c.databaseOnlyMode && c.analyzer != nil {
-		if err := c.analyzer.EnsureConn(ctx, c.schema); err != nil {
+		if err := c.analyzer.EnsureConn(ctx, c.analyzerMigrations()); err != nil {
 			return nil, fmt.Errorf("failed to initialize database connection: %w", err)
 		}
 	}

internal/compiler/compile_test.go

@@ -0,0 +1,43 @@
+package compiler
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/sqlc-dev/sqlc/internal/config"
+	"github.com/sqlc-dev/sqlc/internal/multierr"
+	"github.com/sqlc-dev/sqlc/internal/opts"
+)
+
+func TestParseCatalogManagedAnalyzerRejectsSemanticPsqlCommandsForApply(t *testing.T) {
+	dir := t.TempDir()
+	schema := filepath.Join(dir, "schema.sql")
+	if err := os.WriteFile(schema, []byte("\\include extra.sql\nCREATE TABLE foo (id int);\n"), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	c, err := NewCompiler(config.SQL{
+		Engine: config.EnginePostgreSQL,
+		Schema: []string{schema},
+		Database: &config.Database{
+			Managed: true,
+		},
+	}, config.CombinedSettings{}, opts.Parser{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = c.ParseCatalog([]string{schema})
+	if err == nil {
+		t.Fatal("expected managed analyzer schema preprocessing to reject semantic psql command")
+	}
+	merr, ok := err.(*multierr.Error)
+	if !ok || len(merr.Errs()) != 1 {
+		t.Fatalf("expected one schema error, got %T: %v", err, err)
+	}
+	if !strings.Contains(merr.Errs()[0].Err.Error(), `psql meta-command \include is not supported`) {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}

internal/compiler/engine.go

@@ -27,7 +27,9 @@ type Compiler struct {
 	client   dbmanager.Client
 	selector selector
 
-	schema []string
+	schema      []string
+	applySchema []string
+	warns       []string
 
 	// databaseOnlyMode indicates that the compiler should use database-only analysis
 	// and skip building the internal catalog from schema files (analyzer.database: only)
@@ -125,6 +127,17 @@ func (c *Compiler) ParseCatalog(schema []string) error {
 	return c.parseCatalog(schema)
 }
 
+func (c *Compiler) usesManagedAnalyzer() bool {
+	return c.analyzer != nil && c.conf.Database != nil && c.conf.Database.Managed
+}
+
+func (c *Compiler) analyzerMigrations() []string {
+	if c.usesManagedAnalyzer() {
+		return c.applySchema
+	}
+	return c.schema
+}
+
 func (c *Compiler) ParseQueries(queries []string, o opts.Parser) error {
 	r, err := c.parseQueries(o)
 	if err != nil {
@@ -138,6 +151,12 @@ func (c *Compiler) Result() *Result {
 	return c.result
 }
 
+// Warnings returns a copy of any non-fatal schema preprocessing warnings
+// collected while parsing the catalog.
+func (c *Compiler) Warnings() []string {
+	return append([]string(nil), c.warns...)
+}
+
 func (c *Compiler) Close(ctx context.Context) {
 	if c.analyzer != nil {
 		c.analyzer.Close(ctx)

internal/compiler/parse.go

@@ -93,7 +93,7 @@ func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query,
 		expandedRaw := expandedStmts[0].Raw
 
 		// Use the analyzer to get type information from the database
-		result, err := c.analyzer.Analyze(ctx, expandedRaw, expandedQuery, c.schema, nil)
+		result, err := c.analyzer.Analyze(ctx, expandedRaw, expandedQuery, c.analyzerMigrations(), nil)
 		if err != nil {
 			return nil, err
 		}
@@ -132,7 +132,7 @@ func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query,
 			inference.Query = rawSQL
 		}
 
-		result, err := c.analyzer.Analyze(ctx, raw, inference.Query, c.schema, inference.Named)
+		result, err := c.analyzer.Analyze(ctx, raw, inference.Query, c.analyzerMigrations(), inference.Named)
 		if err != nil {
 			return nil, err
 		}