feat(cli): node selection for dataform compile

cli/api/commands/prune.ts

@@ -20,6 +20,11 @@ export function prune(
     ),
     operations: compiledGraph.operations.filter(action =>
       includedActionNames.has(targetAsReadableString(action.target))
+    ),
+    // `targets` is declarative output only (nothing downstream reads it), but
+    // `compile` prints it, so it has to agree with the filtered action lists.
+    targets: compiledGraph.targets?.filter(target =>
+      includedActionNames.has(targetAsReadableString(target))
     )
   };
 }

cli/index.ts

@@ -5,7 +5,7 @@ import parseDuration from "parse-duration";
 import * as path from "path";
 import yargs from "yargs";
 
-import { build, compile, credentials, init, install, run, test } from "df/cli/api";
+import { build, compile, credentials, init, install, prune, run, test } from "df/cli/api";
 import { CREDENTIALS_FILENAME } from "df/cli/api/commands/credentials";
 import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery";
 import { prettyJsonStringify } from "df/cli/api/utils";
@@ -80,12 +80,29 @@ const fullRefreshOption: INamedOption<yargs.Options> = {
   }
 };
 
+// Splits repeated and comma-separated values into a flat list, e.g.
+// `--actions a,b --actions c` -> ["a", "b", "c"].
+const splitCommas = (raw: string[] | null) => raw.map(value => value.split(",")).flat();
+
+// It would be nice to use yargs' "implies" to implement this, but it doesn't work for some reason.
+const requiresSelection = (
+  name: string,
+  actions: INamedOption<yargs.Options>,
+  tags: INamedOption<yargs.Options>
+): INamedOption<yargs.Options>["check"] => (argv: yargs.Arguments) => {
+  if (argv[name] && !(argv[actions.name] || argv[tags.name])) {
+    throw new Error(
+      `The --${name} flag should only be supplied along with --${actions.name} or --${tags.name}.`
+    );
+  }
+};
+
 const actionsOption: INamedOption<yargs.Options> = {
   name: "actions",
   option: {
     describe: "A list of action names or patterns to run. Can include '*' wildcards.",
     type: "array",
-    coerce: (rawActions: string[] | null) => rawActions.map(actions => actions.split(",")).flat()
+    coerce: splitCommas
   }
 };
 
@@ -94,7 +111,7 @@ const tagsOption: INamedOption<yargs.Options> = {
   option: {
     describe: "A list of tags to filter the actions to run.",
     type: "array",
-    coerce: (rawTags: string[] | null) => rawTags.map(tags => tags.split(",")).flat()
+    coerce: splitCommas
   }
 };
 
@@ -104,14 +121,7 @@ const includeDepsOption: INamedOption<yargs.Options> = {
     describe: "If set, dependencies for selected actions will also be run.",
     type: "boolean"
   },
-  // It would be nice to use yargs' "implies" to implement this, but it doesn't work for some reason.
-  check: (argv: yargs.Arguments) => {
-    if (argv[includeDepsOption.name] && !(argv[actionsOption.name] || argv[tagsOption.name])) {
-      throw new Error(
-        `The --${includeDepsOption.name} flag should only be supplied along with --${actionsOption.name} or --${tagsOption.name}.`
-      );
-    }
-  }
+  check: requiresSelection("include-deps", actionsOption, tagsOption)
 };
 
 const includeDependentsOption: INamedOption<yargs.Options> = {
@@ -120,19 +130,51 @@ const includeDependentsOption: INamedOption<yargs.Options> = {
     describe: "If set, dependents (downstream) for selected actions will also be run.",
     type: "boolean"
   },
-  // It would be nice to use yargs' "implies" to implement this, but it doesn't work for some reason.
-  check: (argv: yargs.Arguments) => {
-    if (
-      argv[includeDependentsOption.name] &&
-      !(argv[actionsOption.name] || argv[tagsOption.name])
-    ) {
-      throw new Error(
-        `The --${includeDependentsOption.name} flag should only be supplied along with --${actionsOption.name} or --${tagsOption.name}.`
-      );
-    }
+  check: requiresSelection("include-dependents", actionsOption, tagsOption)
+};
+
+// `compile` reuses the same prune() filtering as run/build, but these flags only
+// filter the *printed output* -- the whole project still compiles. The `output-`
+// prefix makes that distinction explicit.
+const outputActionsOption: INamedOption<yargs.Options> = {
+  name: "output-actions",
+  option: {
+    // No wildcard support: prune()'s matchPatterns() does exact matching on the
+    // action name or its fully-qualified `database.schema.name`.
+    describe: "A list of action names to filter the compiled output to.",
+    type: "array",
+    coerce: splitCommas
+  }
+};
+
+const outputTagsOption: INamedOption<yargs.Options> = {
+  name: "output-tags",
+  option: {
+    describe: "A list of tags to filter the compiled output to.",
+    type: "array",
+    coerce: splitCommas
   }
 };
 
+const outputIncludeDepsOption: INamedOption<yargs.Options> = {
+  name: "output-include-deps",
+  option: {
+    describe: "If set, dependencies of the selected actions are also included in the output.",
+    type: "boolean"
+  },
+  check: requiresSelection("output-include-deps", outputActionsOption, outputTagsOption)
+};
+
+const outputIncludeDependentsOption: INamedOption<yargs.Options> = {
+  name: "output-include-dependents",
+  option: {
+    describe:
+      "If set, dependents (downstream) of the selected actions are also included in the output.",
+    type: "boolean"
+  },
+  check: requiresSelection("output-include-dependents", outputActionsOption, outputTagsOption)
+};
+
 const credentialsOption: INamedOption<yargs.Options> = {
   name: "credentials",
   option: {
@@ -393,6 +435,10 @@ export function runCli() {
           dotOutputOption,
           timeoutOption,
           quietCompileOption,
+          outputActionsOption,
+          outputTagsOption,
+          outputIncludeDepsOption,
+          outputIncludeDependentsOption,
           {
             name: verboseOptionName,
             option: {
@@ -429,7 +475,24 @@ export function runCli() {
               timeoutMillis: argv[timeoutOption.name] || undefined,
               verbose: argv[verboseOptionName] || false
             });
-            printCompiledGraph(compiledGraph, outputType, argv[quietCompileOption.name]);
+
+            // The whole project must compile (ref() resolution needs every action
+            // registered), but the printed output can be filtered to the selected
+            // action(s) -- mirroring how `run`/`build` prune the graph. We only prune
+            // a clean graph; if compilation produced errors we print the full graph
+            // plus the errors, keeping graph-level errors as-is.
+            const hasSelector =
+              argv[outputActionsOption.name]?.length > 0 || argv[outputTagsOption.name]?.length > 0;
+            const outputGraph =
+              hasSelector && !compiledGraphHasErrors(compiledGraph)
+                ? prune(compiledGraph, {
+                    actions: argv[outputActionsOption.name],
+                    tags: argv[outputTagsOption.name],
+                    includeDependencies: argv[outputIncludeDepsOption.name],
+                    includeDependents: argv[outputIncludeDependentsOption.name]
+                  })
+                : compiledGraph;
+            printCompiledGraph(outputGraph, outputType, argv[quietCompileOption.name]);
             if (compiledGraphHasErrors(compiledGraph)) {
               print("");
               printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]);

cli/index_compile_test.ts

@@ -361,6 +361,143 @@ SELECT 1 as id
   });
 });
 
+suite("compile node selection", ({ afterEach }) => {
+  const tmpDirFixture = new TmpDirFixture(afterEach);
+
+  // Builds a project with three tables: upstream -> midstream -> downstream.
+  async function setupSelectionProject(): Promise<string> {
+    const projectDir = tmpDirFixture.createNewTmpDir();
+    const npmCacheDir = tmpDirFixture.createNewTmpDir();
+
+    await getProcessResult(
+      execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION])
+    );
+
+    const workflowSettingsPath = path.join(projectDir, "workflow_settings.yaml");
+    const workflowSettings = dataform.WorkflowSettings.create(
+      loadYaml(fs.readFileSync(workflowSettingsPath, "utf8"))
+    );
+    delete workflowSettings.dataformCoreVersion;
+    fs.writeFileSync(workflowSettingsPath, dumpYaml(workflowSettings));
+
+    fs.writeFileSync(
+      path.join(projectDir, "package.json"),
+      `{
+  "dependencies":{
+    "@dataform/core": "${version}"
+  }
+}`
+    );
+    await getProcessResult(
+      execFile(npmPath, [
+        "install",
+        "--prefix",
+        projectDir,
+        "--cache",
+        npmCacheDir,
+        corePackageTarPath
+      ])
+    );
+
+    const def = (name: string, contents: string) => {
+      const filePath = path.join(projectDir, "definitions", `${name}.sqlx`);
+      fs.ensureFileSync(filePath);
+      fs.writeFileSync(filePath, contents);
+    };
+    def("upstream", `config { type: "table", tags: ["daily"] }\nSELECT 1 AS id`);
+    def("midstream", `config { type: "table" }\nSELECT * FROM \${ref("upstream")}`);
+    def("downstream", `config { type: "table" }\nSELECT * FROM \${ref("midstream")}`);
+
+    return projectDir;
+  }
+
+  const tableNames = (stdout: string): string[] =>
+    JSON.parse(stdout).tables.map((table: any) => table.target.name).sort();
+
+  test("no selector emits the entire graph", async () => {
+    const projectDir = await setupSelectionProject();
+    const result = await getProcessResult(
+      execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--json"])
+    );
+    expect(result.exitCode, result.stderr).equals(0);
+    expect(tableNames(result.stdout)).deep.equals(["downstream", "midstream", "upstream"]);
+  });
+
+  test("--output-actions filters output to the selected action", async () => {
+    const projectDir = await setupSelectionProject();
+    const result = await getProcessResult(
+      execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--output-actions", "midstream", "--json"])
+    );
+    expect(result.exitCode, result.stderr).equals(0);
+    expect(tableNames(result.stdout)).deep.equals(["midstream"]);
+    // `targets` is printed alongside the actions, so it must be pruned too.
+    expect(JSON.parse(result.stdout).targets.map((target: any) => target.name)).deep.equals([
+      "midstream"
+    ]);
+  });
+
+  test("--output-actions --output-include-deps pulls in upstream dependencies", async () => {
+    const projectDir = await setupSelectionProject();
+    const result = await getProcessResult(
+      execFile(nodePath, [
+        cliEntryPointPath,
+        "compile",
+        projectDir,
+        "--output-actions",
+        "midstream",
+        "--output-include-deps",
+        "--json"
+      ])
+    );
+    expect(result.exitCode, result.stderr).equals(0);
+    expect(tableNames(result.stdout)).deep.equals(["midstream", "upstream"]);
+  });
+
+  test("--output-actions --output-include-dependents pulls in downstream dependents", async () => {
+    const projectDir = await setupSelectionProject();
+    const result = await getProcessResult(
+      execFile(nodePath, [
+        cliEntryPointPath,
+        "compile",
+        projectDir,
+        "--output-actions",
+        "midstream",
+        "--output-include-dependents",
+        "--json"
+      ])
+    );
+    expect(result.exitCode, result.stderr).equals(0);
+    expect(tableNames(result.stdout)).deep.equals(["downstream", "midstream"]);
+  });
+
+  test("--output-tags filters output to actions carrying the tag", async () => {
+    const projectDir = await setupSelectionProject();
+    const result = await getProcessResult(
+      execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--output-tags", "daily", "--json"])
+    );
+    expect(result.exitCode, result.stderr).equals(0);
+    expect(tableNames(result.stdout)).deep.equals(["upstream"]);
+  });
+
+  test("selector matching nothing emits an empty graph and exits zero", async () => {
+    const projectDir = await setupSelectionProject();
+    const result = await getProcessResult(
+      execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--output-actions", "nope", "--json"])
+    );
+    expect(result.exitCode, result.stderr).equals(0);
+    expect(tableNames(result.stdout)).deep.equals([]);
+  });
+
+  test("--output-include-deps without a selector is rejected", async () => {
+    const projectDir = await setupSelectionProject();
+    const result = await getProcessResult(
+      execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--output-include-deps", "--json"])
+    );
+    expect(result.exitCode).not.equals(0);
+    expect(result.stderr).contains("--output-include-deps");
+  });
+});
+
 suite("extension config", ({ afterEach }) => {
   const tmpDirFixture = new TmpDirFixture(afterEach);