diff --git a/DataProvider/Nimblesite.DataProvider.Core/README.md b/DataProvider/Nimblesite.DataProvider.Core/README.md index 0b001904..908f1e0b 100644 --- a/DataProvider/Nimblesite.DataProvider.Core/README.md +++ b/DataProvider/Nimblesite.DataProvider.Core/README.md @@ -5,4 +5,4 @@ The core source generator library. Parses SQL files and generates strongly-typed ## Documentation - Parent README: [DataProvider/README.md](../README.md) -- Migration CLI spec: [docs/specs/migration-cli-spec.md](../../docs/specs/migration-cli-spec.md) +- Migration spec: [docs/specs/migration-spec.md](../../docs/specs/migration-spec.md#74-dataprovidermigrate-cli-mig-cli) diff --git a/DataProvider/README.md b/DataProvider/README.md index c253b3c9..fbb280ed 100644 --- a/DataProvider/README.md +++ b/DataProvider/README.md @@ -152,4 +152,4 @@ SqliteCodeGenerator.GenerateCodeWithMetadata(config: config, /* … */); - [LQL](../Lql/README.md) — cross-database query language that transpiles to SQL - [Migrations](../Migration/README.md) — YAML schema definitions consumed by `DataProviderMigrate` -- Migration CLI spec: [docs/specs/migration-cli-spec.md](../docs/specs/migration-cli-spec.md) +- Migration spec: [docs/specs/migration-spec.md](../docs/specs/migration-spec.md#74-dataprovidermigrate-cli-mig-cli) diff --git a/Migration/DataProviderMigrate/Program.cs b/Migration/DataProviderMigrate/Program.cs index 81e6a3cb..3fc1e9ae 100644 --- a/Migration/DataProviderMigrate/Program.cs +++ b/Migration/DataProviderMigrate/Program.cs @@ -94,6 +94,14 @@ private static int ExecuteMigration(MigrateParseResult.Success args) return 1; } + if (IsSqlServerProvider(args.Provider) && SchemaContainsRls(schema)) + { + // Implements [RLS-MSSQL]. The SQL Server migration package does + // not exist yet, so RLS targeting SQL Server must fail closed. + Console.WriteLine(MigrationError.RlsMssqlUnsupported().Message); + return 1; + } + return args.Provider.ToLowerInvariant() switch { "sqlite" => MigrateSqliteDatabase( @@ -112,6 +120,13 @@ private static int ExecuteMigration(MigrateParseResult.Success args) }; } + private static bool IsSqlServerProvider(string provider) => + provider.Equals("sqlserver", StringComparison.OrdinalIgnoreCase) + || provider.Equals("mssql", StringComparison.OrdinalIgnoreCase); + + private static bool SchemaContainsRls(SchemaDefinition schema) => + schema.Tables.Any(table => table.RowLevelSecurity is not null); + private static int MigrateSqliteDatabase( SchemaDefinition schema, string outputPath, diff --git a/Migration/Nimblesite.DataProvider.Migration.Tests/DataProviderMigrateRlsUnsupportedTests.cs b/Migration/Nimblesite.DataProvider.Migration.Tests/DataProviderMigrateRlsUnsupportedTests.cs new file mode 100644 index 00000000..92e8edcc --- /dev/null +++ b/Migration/Nimblesite.DataProvider.Migration.Tests/DataProviderMigrateRlsUnsupportedTests.cs @@ -0,0 +1,102 @@ +using System.Globalization; + +namespace Nimblesite.DataProvider.Migration.Tests; + +// Tests [RLS-MSSQL] SQL Server package absence guard from docs/specs/rls-spec.md. + +/// +/// CLI tests for unsupported SQL Server RLS migration attempts. +/// +public sealed class DataProviderMigrateRlsUnsupportedTests +{ + [Fact] + public void Migrate_SqlServerProviderWithRlsSchema_ReturnsCanonicalUnsupportedError() + { + var schemaPath = WriteTempSchemaFile(contents: RlsSchemaYaml()); + + try + { + var result = RunMigrate(schemaPath: schemaPath); + + Assert.Equal(expected: 1, actual: result.ExitCode); + Assert.Contains( + expectedSubstring: "MIG-E-RLS-MSSQL-UNSUPPORTED", + actualString: result.Output, + comparisonType: StringComparison.Ordinal + ); + Assert.Contains( + expectedSubstring: "Nimblesite.DataProvider.Migration.SqlServer package does not exist", + actualString: result.Output, + comparisonType: StringComparison.Ordinal + ); + } + finally + { + File.Delete(path: schemaPath); + } + } + + private static (int ExitCode, string Output) RunMigrate(string schemaPath) + { + var originalOut = Console.Out; + using var output = new StringWriter(CultureInfo.InvariantCulture); + Console.SetOut(output); + try + { + var exitCode = DataProviderMigrate.Program.Main( + args: + [ + "migrate", + "--schema", + schemaPath, + "--provider", + "sqlserver", + "--output", + "unused", + ] + ); + return (exitCode, output.ToString()); + } + finally + { + Console.SetOut(originalOut); + } + } + + private static string WriteTempSchemaFile(string contents) + { + var schemaPath = Path.Combine( + Path.GetTempPath(), + string.Create( + CultureInfo.InvariantCulture, + $"dataprovider-rls-unsupported-{Guid.NewGuid():N}.yaml" + ) + ); + File.WriteAllText(path: schemaPath, contents: contents); + return schemaPath; + } + + private static string RlsSchemaYaml() => + """ + name: sqlserver_rls_guard + tables: + - name: documents + schema: public + columns: + - name: id + type: Uuid + isNullable: false + - name: owner_id + type: Uuid + isNullable: false + primaryKey: + columns: + - id + rowLevelSecurity: + policies: + - name: owner_isolation + operations: + - Select + using: owner_id = current_user_id() + """; +} diff --git a/Migration/Nimblesite.DataProvider.Migration.Tests/Nimblesite.DataProvider.Migration.Tests.csproj b/Migration/Nimblesite.DataProvider.Migration.Tests/Nimblesite.DataProvider.Migration.Tests.csproj index 5f0576ed..3b17fbf3 100644 --- a/Migration/Nimblesite.DataProvider.Migration.Tests/Nimblesite.DataProvider.Migration.Tests.csproj +++ b/Migration/Nimblesite.DataProvider.Migration.Tests/Nimblesite.DataProvider.Migration.Tests.csproj @@ -25,6 +25,7 @@ + diff --git a/Migration/README.md b/Migration/README.md index 5fd2f250..5ebb8fc6 100644 --- a/Migration/README.md +++ b/Migration/README.md @@ -202,4 +202,4 @@ Regenerate the database on every build so developers never run migrations manual - [DataProvider](../DataProvider/README.md) — generated extension methods for the tables defined here - [LQL](../Lql/README.md) — write portable queries against the migrated schema -- Migration CLI spec: [docs/specs/migration-cli-spec.md](../docs/specs/migration-cli-spec.md) +- Migration spec: [docs/specs/migration-spec.md](../docs/specs/migration-spec.md#74-dataprovidermigrate-cli-mig-cli) diff --git a/Reporting/spec.md b/Reporting/spec.md index e4efb31b..de563787 100644 --- a/Reporting/spec.md +++ b/Reporting/spec.md @@ -259,12 +259,12 @@ The renderer is a standalone JS bundle. Embed in any page: ## Database Schema -**All database schemas MUST be created using the Migration library with YAML definitions.** Raw SQL for creating database schemas is strictly prohibited. Use `Migration.Cli` with YAML schema files as the single source of truth. +**All database schemas MUST be created using the Migration library with YAML definitions.** Raw SQL for creating database schemas is strictly prohibited. Use `DataProviderMigrate` with YAML schema files as the single source of truth. If the reporting platform requires its own persistence (e.g., for saved reports, scheduled executions), the schema MUST be defined in a `reporting-schema.yaml` file and created via: ```bash -dotnet run --project Migration/Migration.Cli -- --schema reporting-schema.yaml --output reporting.db --provider sqlite +dotnet run --project Migration/DataProviderMigrate/DataProviderMigrate.csproj -- migrate --schema reporting-schema.yaml --output reporting.db --provider sqlite ``` For the MVP, report definitions are loaded from JSON files on disk (no database persistence needed). Future phases will add YAML-migrated schema for saved reports. diff --git a/coverage-thresholds.json b/coverage-thresholds.json index cdf273e2..24555558 100644 --- a/coverage-thresholds.json +++ b/coverage-thresholds.json @@ -50,10 +50,10 @@ "include": "[Nimblesite.Reporting.Engine]*,[Nimblesite.Reporting.Api]*" }, "Lql/lql-lsp-rust": { - "threshold": 83 + "threshold": 88 }, "Lql/LqlExtension": { - "threshold": 38 + "threshold": 39 } } } diff --git a/docs/plans/RLS-PLAN.md b/docs/plans/RLS-PLAN.md index 1134079f..6b92f2c2 100644 --- a/docs/plans/RLS-PLAN.md +++ b/docs/plans/RLS-PLAN.md @@ -85,14 +85,14 @@ Predicates that query other tables (e.g. group membership) MUST use LQL, transpi - [x] Write failing `RlsPredicateTranspiler` unit tests in new `RlsPredicateTranspilerTests.cs` - [x] Make `RlsPredicateTranspiler` tests pass - [x] Implement RLS operation handling in `PostgresDdlGenerator.cs` (Enable, Create, Drop, Disable) -- [ ] Write failing Postgres RLS E2E tests in `PostgresMigrationTests.cs` +- [x] Write failing Postgres RLS E2E tests in `PostgresRlsE2ETests.cs` - [x] Extend `PostgresSchemaInspector.cs` to read `pg_policies` into `RlsPolicySetDefinition` -- [ ] Make Postgres E2E tests pass +- [x] Make Postgres E2E tests pass - [x] Extend `SchemaDiff.Calculate` in `SchemaDiff.cs` with RLS diff logic - [x] Write failing SQLite RLS E2E tests in `SqliteRlsMigrationTests.cs` - [x] Implement `__rls_context` table, trigger generation, and `_secure` view generation in `SqliteDdlGenerator.cs` - [x] Extend `SqliteSchemaInspector.cs` to reverse-map `rls_*` triggers - [x] Make SQLite E2E tests pass -- [ ] Add `MIG-E-RLS-MSSQL-UNSUPPORTED` error guard for SQL Server -- [ ] Run `make ci` -- all tests pass, coverage thresholds maintained -- [ ] Update `Migration/README.md` with RLS usage examples +- [x] Add `MIG-E-RLS-MSSQL-UNSUPPORTED` error guard for SQL Server +- [x] Run `make ci` -- all tests pass, coverage thresholds maintained +- [x] Update `Migration/README.md` with RLS usage examples diff --git a/docs/specs/migration-cli-spec.md b/docs/specs/migration-cli-spec.md deleted file mode 100644 index 2ec05bbe..00000000 --- a/docs/specs/migration-cli-spec.md +++ /dev/null @@ -1,119 +0,0 @@ -# Migration.Cli Specification - -NOTE: leave the JSON serialization/deserialization code as is for now, but deactivate it. The core will eventually offer JSON, but we are focusing on YAML for now. - -## Overview - -`Migration/Migration.Cli/Migration.Cli.csproj` is the **single, canonical CLI tool** for creating databases from schema definitions. All projects that need to spin up a database for code generation MUST use this executable. There is no other way. - -## Architecture - -Migration.Cli contains the DLLs for both SQLite and Postgres migrations. It is database-agnostic at the interface level - callers specify a YAML schema file path, and the CLI handles the rest. - -## Usage - -``` -dotnet run --project Migration/Migration.Cli/Migration.Cli.csproj -- \ - --schema path/to/schema.yaml \ - --output path/to/database.db \ - --provider [sqlite|postgres] -``` - -## Schema Input: YAML Only - -NOTE: leave the JSON serialization/deserialization code as is for now, but deactivate it. The core will eventually offer JSON, but we are focusing on YAML for now. - -The CLI accepts **only YAML schema files**. It does not accept: -- C# code references -- Inline schema definitions -- Project references to schema classes - -If a project defines its schema in C# code (e.g., `ExampleSchema.cs`, `ClinicalSchema.cs`), that schema MUST be serialized to YAML first. The YAML file is then passed to Migration.Cli. - -### Schema-to-YAML Workflow - -1. Schema is defined in a **separate Migrations assembly** (e.g., `MyProject.Migrations/`) with NO dependencies on generated code -2. Build step compiles the Migrations assembly first -3. Migration.Cli `export` subcommand exports C# schema to YAML file -4. Migration.Cli `migrate` subcommand reads YAML and creates database -5. DataProvider code generation runs against the created database -6. Main project (e.g., `MyProject.Api/`) compiles with generated code - -### CRITICAL: Separate Migrations Assemblies - -**Schemas MUST be in separate assemblies to avoid circular build dependencies.** - -**Naming convention: Always use `*.Migrations` suffix, never `*.Schema` or `*BuildDb`.** - -Correct pattern: -``` -MyProject.Migrations/ # Schema definition only, NO generated code deps - └── MyProjectSchema.cs # Defines SchemaDefinition -MyProject.Api/ # References MyProject.Migrations, has generated code - └── Generated/ # DataProvider generated code -``` - -The Migrations assembly: -- Contains ONLY the `SchemaDefinition` class -- References ONLY `Migration` (for schema types) -- Has NO dependencies on generated code -- Can be built BEFORE code generation runs - -The API/main assembly: -- References the Migrations assembly -- Contains generated code from DataProvider -- Is built AFTER code generation - -## Why YAML? - -- **No circular dependencies**: Migration.Cli has zero project references to consumer projects -- **Clean build order**: YAML files are static assets, not compiled code -- **Portable**: Schema can be versioned, diffed, and shared without compilation -- **Single tool**: One CLI handles all schemas for all projects - -## Forbidden Patterns - -- Individual `*BuildDb` projects per consumer (causes circular builds) -- `` in Migration.Cli (circular dependency) -- Multiple CLI tools for database creation -- Hardcoded schema names/switches in the CLI -- CLI tool referencing actual schemas -- **Schema classes in the same project as generated code** (causes circular build deps) -- Migrations assemblies with dependencies on generated code -- Using `*.Schema` or `*BuildDb` naming (must use `*.Migrations`) - -## MSBuild Integration - -Consumer projects call Migration.Cli with `export` then `migrate` subcommands in pre-build targets: - -```xml - - - - - - - - - -``` - -No project references. No schema includes. Just paths to assemblies and YAML files. - -## Build Order (CRITICAL) - -To avoid circular dependencies, the build order MUST be: - -``` -1. Migration/Migration.csproj # Core types -2. MyProject.Migrations/ # Schema definition (refs Migration only) -3. Migration.Cli export # Export schema to YAML -4. Migration.Cli migrate # Create DB from YAML -5. DataProvider code generation # Generate C# from DB -6. MyProject.Api/ # Main project with generated code -``` - -The Migrations assembly MUST NOT reference: -- The API/main project -- Any generated code -- Any project that depends on generated code diff --git a/docs/specs/migration-spec.md b/docs/specs/migration-spec.md index 20cbff7c..c88619d7 100644 --- a/docs/specs/migration-spec.md +++ b/docs/specs/migration-spec.md @@ -14,7 +14,8 @@ 10. [Error Handling](#10-error-handling) 11. [Conformance Requirements](#11-conformance-requirements) 12. [E2E Testing Requirements](#12-e2e-testing-requirements) -13. [Appendices](#13-appendices) +13. [Schema Capture and Metadata](#13-schema-capture-and-metadata) +14. [Appendices](#14-appendices) --- @@ -134,7 +135,7 @@ var schema = Schema.Define("MyApp") ### 4.3 YAML Schema Format -Schema files use YAML format. See [migration-cli-spec.md](migration-cli-spec.md) for CLI usage. The YAML format mirrors the C# records: +Schema files use YAML format. The `DataProviderMigrate` CLI contract is defined in [7.4 DataProviderMigrate CLI](#74-dataprovidermigrate-cli-mig-cli). The YAML format mirrors the C# records: ```yaml name: MyApp @@ -625,6 +626,159 @@ idempotency proofs can verify the constraint was materialized. 5. Return result with applied operations ``` +### 7.4 DataProviderMigrate CLI [MIG-CLI] + +`Migration/DataProviderMigrate/DataProviderMigrate.csproj` is the single, canonical CLI tool for creating databases from schema definitions. All projects that need to spin up a database for code generation MUST use this executable or the packaged `DataProviderMigrate` .NET tool. + +The CLI contains the SQLite and PostgreSQL migration providers. It is database-agnostic at the command surface: callers pass a YAML schema file path, an output database path or connection string, and a provider name. + +#### 7.4.1 Commands [MIG-CLI-COMMANDS] + +Installed tool usage: + +```bash +dotnet DataProviderMigrate migrate \ + --schema path/to/schema.yaml \ + --output path/to/database.db \ + --provider sqlite + +dotnet DataProviderMigrate export \ + --assembly path/to/MyProject.Migrations.dll \ + --type MyProject.Migrations.MyProjectSchema \ + --output path/to/schema.yaml +``` + +Repository-local usage: + +```bash +dotnet run --project Migration/DataProviderMigrate/DataProviderMigrate.csproj -- \ + migrate \ + --schema path/to/schema.yaml \ + --output path/to/database.db \ + --provider sqlite +``` + +`migrate` options: + +| Option | Required | Meaning | +|--------|----------|---------| +| `--schema`, `-s` | Yes | Path to a YAML schema definition file | +| `--output`, `-o` | Yes | SQLite database file path or PostgreSQL connection string | +| `--provider`, `-p` | No | `sqlite` or `postgres`; defaults to `sqlite` | +| `--allow-destructive` | No | Permits destructive drift cleanup operations; off by default | +| `--phase` | No | `all`, `structural`, or `rls`; defaults to `all` | + +`export` options: + +| Option | Required | Meaning | +|--------|----------|---------| +| `--assembly`, `-a` | Yes | Compiled assembly containing the schema type | +| `--type`, `-t` | Yes | Fully qualified schema type name | +| `--output`, `-o` | Yes | YAML file path to write | + +Schema export types MUST expose either a static `Definition` property returning `SchemaDefinition` or a static `Build()` method returning `SchemaDefinition`. + +#### 7.4.2 YAML-Only Migration Input [MIG-CLI-YAML-ONLY] + +The `migrate` command accepts only YAML schema files. It does not accept: + +- C# code references +- Inline schema definitions +- Project references to schema classes +- JSON schema files + +JSON serialization/deserialization code may remain dormant in the core library for future support, but the CLI MUST NOT expose JSON schema input until YAML support is complete and the format is explicitly specified. + +If a project defines its schema in C# code, that schema MUST be exported to YAML first. The YAML file is then passed to `DataProviderMigrate migrate`. + +#### 7.4.3 Schema-to-YAML Workflow [MIG-CLI-SCHEMA-YAML-WORKFLOW] + +Consumer build pipelines that start from C# schema definitions MUST use this order: + +1. Define schema in a separate `*.Migrations` assembly with no generated-code dependencies. +2. Build the migrations assembly first. +3. Run `DataProviderMigrate export` to serialize the C# schema to YAML. +4. Run `DataProviderMigrate migrate` to create or update the target database from YAML. +5. Run DataProvider code generation against the created database. +6. Build the main project with generated code included. + +#### 7.4.4 Separate Migrations Assemblies [MIG-CLI-MIGRATIONS-ASSEMBLY] + +Schemas MUST live in separate migrations assemblies to avoid circular build dependencies. + +Naming convention: use the `*.Migrations` suffix. Do not use `*.Schema` or `*BuildDb`. + +Correct pattern: + +```text +MyProject.Migrations/ + MyProjectSchema.cs # Defines SchemaDefinition + +MyProject.Api/ + Generated/ # DataProvider generated code +``` + +The migrations assembly: + +- Contains only schema definition code. +- References only migration schema types and their direct dependencies. +- Has no dependencies on generated code. +- Builds before code generation runs. + +The API or main assembly: + +- References the migrations assembly only when it needs the schema at runtime. +- Contains DataProvider generated code. +- Builds after code generation. + +#### 7.4.5 Build Integration [MIG-CLI-BUILD-INTEGRATION] + +Consumer projects may wire export and migrate into MSBuild pre-build targets: + +```xml + + + + + + + +``` + +There are no project references from the CLI to consumer schemas, no schema source includes, and no hardcoded schema names or switches in the CLI. + +#### 7.4.6 Required Build Order [MIG-CLI-BUILD-ORDER] + +To avoid circular dependencies, builds that need generated data access code MUST follow this order: + +```text +1. Migration/Nimblesite.DataProvider.Migration.Core +2. MyProject.Migrations +3. DataProviderMigrate export +4. DataProviderMigrate migrate +5. DataProvider code generation +6. MyProject.Api +``` + +The migrations assembly MUST NOT reference: + +- The API or main project. +- Any generated code. +- Any project that depends on generated code. + +#### 7.4.7 Forbidden CLI Patterns [MIG-CLI-FORBIDDEN] + +The following patterns are not conformant: + +- Individual `*BuildDb` projects per consumer. +- `` in the CLI project. +- Multiple CLI tools for database creation. +- Hardcoded schema names or project-specific switches in the CLI. +- CLI references to consumer schema projects. +- Schema classes in the same project as generated code. +- Migrations assemblies with dependencies on generated code. +- `*.Schema` or `*BuildDb` naming for migrations projects. + --- ## 8. Diff Engine