diff --git a/cli/api/commands/prune.ts b/cli/api/commands/prune.ts index 71358ea0b..b92c1e94d 100644 --- a/cli/api/commands/prune.ts +++ b/cli/api/commands/prune.ts @@ -20,6 +20,11 @@ export function prune( ), operations: compiledGraph.operations.filter(action => includedActionNames.has(targetAsReadableString(action.target)) + ), + // `targets` is declarative output only (nothing downstream reads it), but + // `compile` prints it, so it has to agree with the filtered action lists. + targets: compiledGraph.targets?.filter(target => + includedActionNames.has(targetAsReadableString(target)) ) }; } diff --git a/cli/index.ts b/cli/index.ts index 452d71f5f..b3fb7e05c 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -5,7 +5,7 @@ import parseDuration from "parse-duration"; import * as path from "path"; import yargs from "yargs"; -import { build, compile, credentials, init, install, run, test } from "df/cli/api"; +import { build, compile, credentials, init, install, prune, run, test } from "df/cli/api"; import { CREDENTIALS_FILENAME } from "df/cli/api/commands/credentials"; import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; import { prettyJsonStringify } from "df/cli/api/utils"; @@ -80,12 +80,29 @@ const fullRefreshOption: INamedOption = { } }; +// Splits repeated and comma-separated values into a flat list, e.g. +// `--actions a,b --actions c` -> ["a", "b", "c"]. +const splitCommas = (raw: string[] | null) => raw.map(value => value.split(",")).flat(); + +// It would be nice to use yargs' "implies" to implement this, but it doesn't work for some reason. +const requiresSelection = ( + name: string, + actions: INamedOption, + tags: INamedOption +): INamedOption["check"] => (argv: yargs.Arguments) => { + if (argv[name] && !(argv[actions.name] || argv[tags.name])) { + throw new Error( + `The --${name} flag should only be supplied along with --${actions.name} or --${tags.name}.` + ); + } +}; + const actionsOption: INamedOption = { name: "actions", option: { describe: "A list of action names or patterns to run. Can include '*' wildcards.", type: "array", - coerce: (rawActions: string[] | null) => rawActions.map(actions => actions.split(",")).flat() + coerce: splitCommas } }; @@ -94,7 +111,7 @@ const tagsOption: INamedOption = { option: { describe: "A list of tags to filter the actions to run.", type: "array", - coerce: (rawTags: string[] | null) => rawTags.map(tags => tags.split(",")).flat() + coerce: splitCommas } }; @@ -104,14 +121,7 @@ const includeDepsOption: INamedOption = { describe: "If set, dependencies for selected actions will also be run.", type: "boolean" }, - // It would be nice to use yargs' "implies" to implement this, but it doesn't work for some reason. - check: (argv: yargs.Arguments) => { - if (argv[includeDepsOption.name] && !(argv[actionsOption.name] || argv[tagsOption.name])) { - throw new Error( - `The --${includeDepsOption.name} flag should only be supplied along with --${actionsOption.name} or --${tagsOption.name}.` - ); - } - } + check: requiresSelection("include-deps", actionsOption, tagsOption) }; const includeDependentsOption: INamedOption = { @@ -120,19 +130,51 @@ const includeDependentsOption: INamedOption = { describe: "If set, dependents (downstream) for selected actions will also be run.", type: "boolean" }, - // It would be nice to use yargs' "implies" to implement this, but it doesn't work for some reason. - check: (argv: yargs.Arguments) => { - if ( - argv[includeDependentsOption.name] && - !(argv[actionsOption.name] || argv[tagsOption.name]) - ) { - throw new Error( - `The --${includeDependentsOption.name} flag should only be supplied along with --${actionsOption.name} or --${tagsOption.name}.` - ); - } + check: requiresSelection("include-dependents", actionsOption, tagsOption) +}; + +// `compile` reuses the same prune() filtering as run/build, but these flags only +// filter the *printed output* -- the whole project still compiles. The `output-` +// prefix makes that distinction explicit. +const outputActionsOption: INamedOption = { + name: "output-actions", + option: { + // No wildcard support: prune()'s matchPatterns() does exact matching on the + // action name or its fully-qualified `database.schema.name`. + describe: "A list of action names to filter the compiled output to.", + type: "array", + coerce: splitCommas + } +}; + +const outputTagsOption: INamedOption = { + name: "output-tags", + option: { + describe: "A list of tags to filter the compiled output to.", + type: "array", + coerce: splitCommas } }; +const outputIncludeDepsOption: INamedOption = { + name: "output-include-deps", + option: { + describe: "If set, dependencies of the selected actions are also included in the output.", + type: "boolean" + }, + check: requiresSelection("output-include-deps", outputActionsOption, outputTagsOption) +}; + +const outputIncludeDependentsOption: INamedOption = { + name: "output-include-dependents", + option: { + describe: + "If set, dependents (downstream) of the selected actions are also included in the output.", + type: "boolean" + }, + check: requiresSelection("output-include-dependents", outputActionsOption, outputTagsOption) +}; + const credentialsOption: INamedOption = { name: "credentials", option: { @@ -393,6 +435,10 @@ export function runCli() { dotOutputOption, timeoutOption, quietCompileOption, + outputActionsOption, + outputTagsOption, + outputIncludeDepsOption, + outputIncludeDependentsOption, { name: verboseOptionName, option: { @@ -429,7 +475,24 @@ export function runCli() { timeoutMillis: argv[timeoutOption.name] || undefined, verbose: argv[verboseOptionName] || false }); - printCompiledGraph(compiledGraph, outputType, argv[quietCompileOption.name]); + + // The whole project must compile (ref() resolution needs every action + // registered), but the printed output can be filtered to the selected + // action(s) -- mirroring how `run`/`build` prune the graph. We only prune + // a clean graph; if compilation produced errors we print the full graph + // plus the errors, keeping graph-level errors as-is. + const hasSelector = + argv[outputActionsOption.name]?.length > 0 || argv[outputTagsOption.name]?.length > 0; + const outputGraph = + hasSelector && !compiledGraphHasErrors(compiledGraph) + ? prune(compiledGraph, { + actions: argv[outputActionsOption.name], + tags: argv[outputTagsOption.name], + includeDependencies: argv[outputIncludeDepsOption.name], + includeDependents: argv[outputIncludeDependentsOption.name] + }) + : compiledGraph; + printCompiledGraph(outputGraph, outputType, argv[quietCompileOption.name]); if (compiledGraphHasErrors(compiledGraph)) { print(""); printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]); diff --git a/cli/index_compile_test.ts b/cli/index_compile_test.ts index 4f82799b7..c27ac47eb 100644 --- a/cli/index_compile_test.ts +++ b/cli/index_compile_test.ts @@ -361,6 +361,143 @@ SELECT 1 as id }); }); +suite("compile node selection", ({ afterEach }) => { + const tmpDirFixture = new TmpDirFixture(afterEach); + + // Builds a project with three tables: upstream -> midstream -> downstream. + async function setupSelectionProject(): Promise { + const projectDir = tmpDirFixture.createNewTmpDir(); + const npmCacheDir = tmpDirFixture.createNewTmpDir(); + + await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + ); + + const workflowSettingsPath = path.join(projectDir, "workflow_settings.yaml"); + const workflowSettings = dataform.WorkflowSettings.create( + loadYaml(fs.readFileSync(workflowSettingsPath, "utf8")) + ); + delete workflowSettings.dataformCoreVersion; + fs.writeFileSync(workflowSettingsPath, dumpYaml(workflowSettings)); + + fs.writeFileSync( + path.join(projectDir, "package.json"), + `{ + "dependencies":{ + "@dataform/core": "${version}" + } +}` + ); + await getProcessResult( + execFile(npmPath, [ + "install", + "--prefix", + projectDir, + "--cache", + npmCacheDir, + corePackageTarPath + ]) + ); + + const def = (name: string, contents: string) => { + const filePath = path.join(projectDir, "definitions", `${name}.sqlx`); + fs.ensureFileSync(filePath); + fs.writeFileSync(filePath, contents); + }; + def("upstream", `config { type: "table", tags: ["daily"] }\nSELECT 1 AS id`); + def("midstream", `config { type: "table" }\nSELECT * FROM \${ref("upstream")}`); + def("downstream", `config { type: "table" }\nSELECT * FROM \${ref("midstream")}`); + + return projectDir; + } + + const tableNames = (stdout: string): string[] => + JSON.parse(stdout).tables.map((table: any) => table.target.name).sort(); + + test("no selector emits the entire graph", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--json"]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals(["downstream", "midstream", "upstream"]); + }); + + test("--output-actions filters output to the selected action", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--output-actions", "midstream", "--json"]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals(["midstream"]); + // `targets` is printed alongside the actions, so it must be pruned too. + expect(JSON.parse(result.stdout).targets.map((target: any) => target.name)).deep.equals([ + "midstream" + ]); + }); + + test("--output-actions --output-include-deps pulls in upstream dependencies", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [ + cliEntryPointPath, + "compile", + projectDir, + "--output-actions", + "midstream", + "--output-include-deps", + "--json" + ]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals(["midstream", "upstream"]); + }); + + test("--output-actions --output-include-dependents pulls in downstream dependents", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [ + cliEntryPointPath, + "compile", + projectDir, + "--output-actions", + "midstream", + "--output-include-dependents", + "--json" + ]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals(["downstream", "midstream"]); + }); + + test("--output-tags filters output to actions carrying the tag", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--output-tags", "daily", "--json"]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals(["upstream"]); + }); + + test("selector matching nothing emits an empty graph and exits zero", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--output-actions", "nope", "--json"]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals([]); + }); + + test("--output-include-deps without a selector is rejected", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--output-include-deps", "--json"]) + ); + expect(result.exitCode).not.equals(0); + expect(result.stderr).contains("--output-include-deps"); + }); +}); + suite("extension config", ({ afterEach }) => { const tmpDirFixture = new TmpDirFixture(afterEach);