From 6a2555099e15e4d9986fecf66439fc54f6d83bd8 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Tue, 4 Aug 2026 17:09:23 +0800 Subject: [PATCH] feat(CI): Introduce xtask, and use it for rust CI's `cargo check ...` and `cargo test ...` --- .cargo/config.toml | 20 + .github/workflows/rust.yml | 122 +++---- Cargo.lock | 4 + Cargo.toml | 1 + xtask/Cargo.toml | 26 ++ xtask/src/ci_steps.rs | 728 +++++++++++++++++++++++++++++++++++++ xtask/src/main.rs | 134 +++++++ 7 files changed, 964 insertions(+), 71 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 xtask/Cargo.toml create mode 100644 xtask/src/ci_steps.rs create mode 100644 xtask/src/main.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000000000..9dae805eb1981 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[alias] +xtask = "run --quiet --locked --package datafusion-xtask --" +ci-step = "run --quiet --locked --package datafusion-xtask -- ci step" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1f3f5269de04c..7c386ba237f1c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -63,11 +63,8 @@ jobs: shared-key: "amd-ci-check" # this job uses it's own cache becase check has a separate cache and we need it to be fast as it blocks other jobs save-if: ${{ github.ref_name == 'main' }} - name: Prepare cargo build - run: | - # Adding `--locked` here to assert that the `Cargo.lock` file is up to - # date with the manifest. When this fails, please make sure to commit - # the changes to `Cargo.lock` after building with the updated manifest. - cargo check --profile ci --workspace --all-targets --features integration-tests --locked + # `--locked` asserts that Cargo.lock is up to date with the manifest. + run: cargo xtask ci step check workspace # Check datafusion-common features # @@ -86,12 +83,12 @@ jobs: with: rust-version: stable - name: Check datafusion-common (default features) - run: cargo check --profile ci --all-targets -p datafusion-common + run: cargo xtask ci step check datafusion-common default # # Note: Only check libraries (not --all-targets) to cover end user APIs # - name: Check datafusion-common (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion-common + run: cargo xtask ci step check datafusion-common no-default # Note: don't check other feature flags as datafusion-common is not typically used standalone # Check datafusion-substrait features @@ -116,20 +113,20 @@ jobs: save-if: false # set in linux-test shared-key: "amd-ci" - name: Check datafusion-substrait (default features) - run: cargo check --profile ci --all-targets -p datafusion-substrait + run: cargo xtask ci step check datafusion-substrait default # # Note: Only check libraries (not --all-targets) to cover end user APIs # - name: Check datafusion-substrait (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion-substrait + run: cargo xtask ci step check datafusion-substrait no-default - name: Check datafusion-substrait (physical) - run: cargo check --profile ci --no-default-features -p datafusion-substrait --features=physical + run: cargo xtask ci step check datafusion-substrait physical - name: Install cmake run: | # note the builder setup runs apt-get update / installs protobuf compiler apt-get install -y cmake - name: Check datafusion-substrait (protoc) - run: cargo check --profile ci --no-default-features -p datafusion-substrait --features=protoc + run: cargo xtask ci step check datafusion-substrait protoc # Check datafusion-proto features # @@ -149,18 +146,18 @@ jobs: with: rust-version: stable - name: Check datafusion-proto (default features) - run: cargo check --profile ci --all-targets -p datafusion-proto + run: cargo xtask ci step check datafusion-proto default # # Note: Only check libraries (not --all-targets) to cover end user APIs # - name: Check datafusion-proto (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion-proto + run: cargo xtask ci step check datafusion-proto no-default - name: Check datafusion-proto (json) - run: cargo check --profile ci --no-default-features -p datafusion-proto --features=json + run: cargo xtask ci step check datafusion-proto json - name: Check datafusion-proto (parquet) - run: cargo check --profile ci --no-default-features -p datafusion-proto --features=parquet + run: cargo xtask ci step check datafusion-proto parquet - name: Check datafusion-proto (avro) - run: cargo check --profile ci --no-default-features -p datafusion-proto --features=avro + run: cargo xtask ci step check datafusion-proto avro # Check datafusion-ffi features # @@ -180,7 +177,7 @@ jobs: with: rust-version: stable - name: Check datafusion-ffi (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion-ffi + run: cargo xtask ci step check datafusion-ffi no-default # Check datafusion crate features @@ -206,48 +203,48 @@ jobs: save-if: false # set in linux-test shared-key: "amd-ci" - name: Check datafusion (default features) - run: cargo check --profile ci --all-targets -p datafusion + run: cargo xtask ci step check datafusion default # # Note: Only check libraries (not --all-targets) to cover end user APIs # - name: Check datafusion (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion + run: cargo xtask ci step check datafusion no-default - name: Check datafusion (nested_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=nested_expressions + run: cargo xtask ci step check datafusion nested_expressions - name: Check datafusion (array_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=array_expressions + run: cargo xtask ci step check datafusion array_expressions - name: Check datafusion (avro) - run: cargo check --profile ci --no-default-features -p datafusion --features=avro + run: cargo xtask ci step check datafusion avro - name: Check datafusion (backtrace) - run: cargo check --profile ci --no-default-features -p datafusion --features=backtrace + run: cargo xtask ci step check datafusion backtrace - name: Check datafusion (compression) - run: cargo check --profile ci --no-default-features -p datafusion --features=compression + run: cargo xtask ci step check datafusion compression - name: Check datafusion (crypto_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=crypto_expressions + run: cargo xtask ci step check datafusion crypto_expressions - name: Check datafusion (datetime_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=datetime_expressions + run: cargo xtask ci step check datafusion datetime_expressions - name: Check datafusion (encoding_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=encoding_expressions + run: cargo xtask ci step check datafusion encoding_expressions - name: Check datafusion (force_hash_collisions) - run: cargo check --profile ci --no-default-features -p datafusion --features=force_hash_collisions + run: cargo xtask ci step check datafusion force_hash_collisions - name: Check datafusion (math_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=math_expressions + run: cargo xtask ci step check datafusion math_expressions - name: Check datafusion (parquet) - run: cargo check --profile ci --no-default-features -p datafusion --features=parquet + run: cargo xtask ci step check datafusion parquet - name: Check datafusion (regex_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=regex_expressions + run: cargo xtask ci step check datafusion regex_expressions - name: Check datafusion (recursive_protection) - run: cargo check --profile ci --no-default-features -p datafusion --features=recursive_protection + run: cargo xtask ci step check datafusion recursive_protection - name: Check datafusion (serde) - run: cargo check --profile ci --no-default-features -p datafusion --features=serde + run: cargo xtask ci step check datafusion serde - name: Check datafusion (sql) - run: cargo check --profile ci --no-default-features -p datafusion --features=sql + run: cargo xtask ci step check datafusion sql - name: Check datafusion (string_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=string_expressions + run: cargo xtask ci step check datafusion string_expressions - name: Check datafusion (unicode_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=unicode_expressions + run: cargo xtask ci step check datafusion unicode_expressions - name: Check parquet encryption (parquet_encryption) - run: cargo check --profile ci --no-default-features -p datafusion --features=parquet_encryption + run: cargo xtask ci step check datafusion parquet_encryption # Check datafusion-functions crate features # @@ -266,26 +263,26 @@ jobs: with: rust-version: stable - name: Check datafusion-functions (default features) - run: cargo check --profile ci --all-targets -p datafusion-functions + run: cargo xtask ci step check datafusion-functions default # # Note: Only check libraries (not --all-targets) to cover end user APIs # - name: Check datafusion-functions (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion-functions + run: cargo xtask ci step check datafusion-functions no-default - name: Check datafusion-functions (crypto_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=crypto_expressions + run: cargo xtask ci step check datafusion-functions crypto_expressions - name: Check datafusion-functions (datetime_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=datetime_expressions + run: cargo xtask ci step check datafusion-functions datetime_expressions - name: Check datafusion-functions (encoding_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=encoding_expressions + run: cargo xtask ci step check datafusion-functions encoding_expressions - name: Check datafusion-functions (math_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=math_expressions + run: cargo xtask ci step check datafusion-functions math_expressions - name: Check datafusion-functions (regex_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=regex_expressions + run: cargo xtask ci step check datafusion-functions regex_expressions - name: Check datafusion-functions (string_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=string_expressions + run: cargo xtask ci step check datafusion-functions string_expressions - name: Check datafusion-functions (unicode_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=unicode_expressions + run: cargo xtask ci step check datafusion-functions unicode_expressions # Library and integration tests linux-test: @@ -320,19 +317,7 @@ jobs: - name: Run tests (excluding doctests and datafusion-cli) env: RUST_BACKTRACE: 1 - run: | - cargo llvm-cov \ - --profile ci \ - --exclude datafusion-examples \ - --exclude ffi_example_table_provider \ - --exclude datafusion-cli \ - --workspace \ - --lib \ - --tests \ - --bins \ - --features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait \ - --codecov \ - --output-path target/codecov.json + run: cargo xtask ci step test workspace - parallel: - name: Verify Working Directory Clean run: git diff --exit-code @@ -378,7 +363,7 @@ jobs: AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword TEST_STORAGE_INTEGRATION: 1 AWS_ALLOW_HTTP: true - run: cargo test --features backtrace --profile ci -p datafusion-cli --lib --tests --bins + run: cargo xtask ci step test cli - name: Verify Working Directory Clean run: git diff --exit-code @@ -431,7 +416,7 @@ jobs: with: rust-version: stable - name: Run doctests - run: cargo test --profile ci --doc --features avro,json + run: cargo xtask ci step test doctest - name: Verify Working Directory Clean run: git diff --exit-code @@ -505,11 +490,8 @@ jobs: mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data - name: Verify that benchmark queries return expected results run: | - # increase stack size to fix stack overflow - export RUST_MIN_STACK=20971520 - export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data` - cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1 - INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait --profile ci --package datafusion-sqllogictest --test sqllogictests + cargo xtask ci step test benchmark-plan + cargo xtask ci step test benchmark-sqllogic - name: Verify Working Directory Clean run: git diff --exit-code @@ -544,9 +526,7 @@ jobs: with: rust-version: stable - name: Run sqllogictest - run: | - cd datafusion/sqllogictest - PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo test --features backtrace --profile ci --features=postgres --test sqllogictests + run: cargo xtask ci step test postgres env: # use postgres for the host here because we have specified a container for the job POSTGRES_HOST: postgres @@ -573,7 +553,7 @@ jobs: # command cannot be run for all the .slt files. Run it for just one that works (limit.slt) # until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed # and this command can be run without filters. - run: cargo test -p datafusion-sqllogictest --test sqllogictests --features substrait -- --substrait-round-trip limit.slt + run: cargo xtask ci step test substrait # Temporarily commenting out the Windows flow, the reason is enormously slow running build # Waiting for new Windows 2025 github runner @@ -611,7 +591,7 @@ jobs: uses: ./.github/actions/setup-macos-aarch64-builder - name: Run datafusion-ffi tests shell: bash - run: cargo test --profile ci -p datafusion-ffi --lib --tests --features integration-tests + run: cargo xtask ci step test ffi vendor: name: Verify Vendored Code diff --git a/Cargo.lock b/Cargo.lock index 619fee7603629..50ee962f6f38c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2733,6 +2733,10 @@ dependencies = [ "wasm-bindgen-test", ] +[[package]] +name = "datafusion-xtask" +version = "0.1.0" + [[package]] name = "deranged" version = "0.5.8" diff --git a/Cargo.toml b/Cargo.toml index 87c23cc456651..71db43df33e92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ members = [ "benchmarks", "datafusion/macros", "datafusion/doc", + "xtask", ] exclude = ["dev/depcheck"] resolver = "2" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml new file mode 100644 index 0000000000000..ab3670cf41d8f --- /dev/null +++ b/xtask/Cargo.toml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "datafusion-xtask" +version = "0.1.0" +edition.workspace = true +license.workspace = true +publish = false + +[lints] +workspace = true diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rs new file mode 100644 index 0000000000000..9521522d48e94 --- /dev/null +++ b/xtask/src/ci_steps.rs @@ -0,0 +1,728 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! CI steps that can be inspected or run through `cargo xtask`. + +use crate::Result; +use std::env; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +type StepResult = std::result::Result; +type StepRunner = fn(&StepContext, &[String]) -> StepResult; + +const CI_COMMAND: &str = "cargo xtask ci step"; +const CI_SHORTCUT: &str = "cargo ci-step"; + +/// Metadata and implementation for one CI step. +/// +/// Adding an entry to `CI_STEPS` makes the step dispatchable and includes its +/// usage and description in the generated help text. +struct StepInfo { + command: &'static str, + help_usage: &'static str, + help_examples: &'static [&'static str], + help_description: &'static str, + error_message: &'static str, + runner: StepRunner, +} + +static CI_STEPS: &[StepInfo] = &[ + StepInfo { + command: "check", + help_usage: "check [default|no-default|feature]", + help_examples: &["check workspace", "check datafusion default"], + help_description: "Check workspace or package compilation", + error_message: "Cargo check step failed", + runner: StepContext::run_check, + }, + StepInfo { + command: "test", + help_usage: "test ", + help_examples: &["test cli", "test workspace"], + help_description: "Run a CI test suite", + error_message: "Cargo test step failed", + runner: StepContext::run_test, + }, +]; + +pub(crate) fn help() -> String { + let mut help = format!( + "DataFusion CI commands\n\nUsage:\n {CI_COMMAND} [args]\n\nExamples:\n", + ); + for step in CI_STEPS { + if let Some(example) = step.help_examples.first() { + help.push_str(&format!(" {CI_COMMAND} {example}\n")); + } + } + help.push_str(&format!(" {CI_COMMAND} explain test workspace\n")); + + let command_width = CI_STEPS + .iter() + .map(|step| step.command.len()) + .max() + .unwrap_or_default(); + help.push_str("\nAvailable steps:\n"); + for step in CI_STEPS { + help.push_str(&format!( + " {:command_width$} {}\n", + step.command, step.help_description + )); + } + + help.push_str(&format!( + "\nShortcut:\n # `{CI_SHORTCUT}` is short for `{CI_COMMAND}`.\n {CI_SHORTCUT} check workspace\n\nFor more details:\n {CI_COMMAND} check --help\n" + )); + help +} + +fn step_help(step: &StepInfo) -> String { + let mut help = format!( + "DataFusion CI command: {}\n\n{}\n\nUsage:\n {CI_COMMAND} {}\n\nExamples:\n", + step.command, step.help_description, step.help_usage, + ); + for example in step.help_examples { + help.push_str(&format!(" {CI_COMMAND} {example}\n")); + } + if let Some(example) = step.help_examples.first() { + help.push_str(&format!( + "\nUse 'explain' to show the full command:\n {CI_COMMAND} explain {example}\n" + )); + } + help +} + +fn find_step(command: &str) -> Result<&'static StepInfo> { + CI_STEPS + .iter() + .find(|step| step.command == command) + .ok_or_else(|| format!("unknown CI step `{command}`")) +} + +pub(crate) fn is_help_arg(arg: &str) -> bool { + matches!(arg, "help" | "-h" | "--help") +} + +pub(crate) fn run(root: &Path, args: &[String]) -> Result<()> { + StepContext::new(root).run(args) +} + +struct StepContext { + root: PathBuf, +} + +impl StepContext { + fn new(root: &Path) -> Self { + Self { + root: root.to_path_buf(), + } + } + + fn run(&self, args: &[String]) -> Result<()> { + match args { + [] => { + print!("{}", help()); + Ok(()) + } + [help_arg] if is_help_arg(help_arg) => { + print!("{}", help()); + Ok(()) + } + [step_name, help_arg] if is_help_arg(help_arg) => { + print!("{}", step_help(find_step(step_name)?)); + Ok(()) + } + step_args => { + let (action, step, command) = self.ci_step(step_args)?; + match action { + StepAction::Execute => command.execute(step.error_message), + StepAction::Explain => { + command.explain(); + Ok(()) + } + } + } + } + } + + /// Parses a CI step into an action and its complete command description. + /// Keeping execution out of this method guarantees `explain` and execution + /// use exactly the same program, arguments, environment, and directory. + fn ci_step( + &self, + args: &[String], + ) -> Result<(StepAction, &'static StepInfo, CiCommand)> { + let (action, args) = match args.split_first() { + Some((arg, args)) if arg == "explain" => (StepAction::Explain, args), + _ => (StepAction::Execute, args), + }; + let Some((step, args)) = args.split_first() else { + return Err(format!("missing CI step\n\n{}", help())); + }; + + let step = find_step(step)?; + let command = (step.runner)(self, args).map_err(|error| error.render(step))?; + Ok((action, step, command)) + } + + fn run_check(&self, args: &[String]) -> StepResult { + let args = args.iter().map(String::as_str).collect::>(); + let mut command = self.cargo(); + command.args(["check", "--profile", "ci"]); + + match args.as_slice() { + ["workspace"] => { + command.args([ + "--workspace", + "--all-targets", + "--features", + "integration-tests", + "--locked", + ]); + } + [package, "default"] => { + command.args(["--all-targets", "-p", package]); + } + [package, "no-default"] => { + command.args(["--no-default-features", "-p", package]); + } + [package, feature] => { + command.args([ + "--no-default-features", + "-p", + package, + "--features", + feature, + ]); + } + _ => return Err(StepError::Usage), + } + + Ok(command) + } + + fn run_test(&self, args: &[String]) -> StepResult { + let [variant] = args else { + return Err(StepError::Usage); + }; + + let mut command = self.cargo(); + match variant.as_str() { + "workspace" => { + command.args([ + "llvm-cov", + "--profile", + "ci", + "--exclude", + "datafusion-examples", + "--exclude", + "ffi_example_table_provider", + "--exclude", + "datafusion-cli", + "--workspace", + "--lib", + "--tests", + "--bins", + "--features", + "serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait", + "--codecov", + "--output-path", + "target/codecov.json", + ]); + } + "cli" => { + command.args([ + "test", + "--features", + "backtrace", + "--profile", + "ci", + "-p", + "datafusion-cli", + "--lib", + "--tests", + "--bins", + ]); + } + "doctest" => { + command.args([ + "test", + "--profile", + "ci", + "--doc", + "--features", + "avro,json", + ]); + } + "ffi" => { + command.args([ + "test", + "--profile", + "ci", + "-p", + "datafusion-ffi", + "--lib", + "--tests", + "--features", + "integration-tests", + ]); + } + "benchmark-plan" => { + command + .args([ + "test", + "plan_q", + "--package", + "datafusion-benchmarks", + "--profile", + "ci", + "--features=ci", + "--", + "--test-threads=1", + ]) + .env("RUST_MIN_STACK", "20971520") + .env("TPCH_DATA", self.tpch_data_dir()); + } + "benchmark-sqllogic" => { + command + .args([ + "test", + "--features", + "backtrace,parquet_encryption,substrait", + "--profile", + "ci", + "--package", + "datafusion-sqllogictest", + "--test", + "sqllogictests", + ]) + .env("RUST_MIN_STACK", "20971520") + .env("TPCH_DATA", self.tpch_data_dir()) + .env("INCLUDE_TPCH", "true"); + } + "postgres" => { + let host = required_env("POSTGRES_HOST").map_err(StepError::Message)?; + let port = required_env("POSTGRES_PORT").map_err(StepError::Message)?; + let uri = format!("postgresql://postgres:postgres@{host}:{port}/db_test"); + command + .args([ + "test", + "--features", + "backtrace", + "--profile", + "ci", + "--features=postgres", + "--test", + "sqllogictests", + ]) + .current_dir(self.root.join("datafusion/sqllogictest")) + .env("PG_COMPAT", "true") + .env("PG_URI", uri); + } + "substrait" => { + command.args([ + "test", + "-p", + "datafusion-sqllogictest", + "--test", + "sqllogictests", + "--features", + "substrait", + "--", + "--substrait-round-trip", + "limit.slt", + ]); + } + _ => return Err(StepError::Usage), + } + + Ok(command) + } + + fn cargo(&self) -> CiCommand { + CiCommand::new("cargo", &self.root) + } + + fn tpch_data_dir(&self) -> PathBuf { + self.root + .join("datafusion/sqllogictest/test_files/tpch/data") + } +} + +#[derive(Debug, PartialEq)] +enum StepAction { + Execute, + Explain, +} + +#[derive(Debug)] +enum StepError { + Usage, + Message(String), +} + +impl StepError { + fn render(self, step: &StepInfo) -> String { + match self { + Self::Usage => format!("usage: {CI_COMMAND} {}", step.help_usage), + Self::Message(message) => message, + } + } +} + +/// A complete process description shared by explanation and execution. +struct CiCommand { + command: Command, +} + +impl CiCommand { + fn new(program: impl AsRef, current_dir: impl AsRef) -> Self { + let mut command = Command::new(program); + command.current_dir(current_dir); + Self { command } + } + + fn args(&mut self, args: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + self.command.args(args); + self + } + + fn env(&mut self, key: K, value: V) -> &mut Self + where + K: AsRef, + V: AsRef, + { + self.command.env(key, value); + self + } + + fn current_dir(&mut self, directory: impl AsRef) -> &mut Self { + self.command.current_dir(directory); + self + } + + fn explain(&self) { + println!("{}", self.full_command()); + } + + fn execute(mut self, error_message: &str) -> Result<()> { + println!("+ {}", self.full_command()); + let status = self + .command + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status() + .map_err(|error| format!("{error_message}: {error}"))?; + + if status.success() { + Ok(()) + } else { + Err(format!("{error_message}: {status}")) + } + } + + fn full_command(&self) -> String { + let current_dir = self.command.get_current_dir(); + let mut output = String::new(); + + if let Some(current_dir) = current_dir { + push_shell_line( + &mut output, + "", + &format!("cd {} &&", shell_quote(current_dir.as_os_str())), + true, + ); + } + + for (key, value) in self.command.get_envs() { + let Some(value) = value else { + continue; + }; + push_shell_line( + &mut output, + "", + &format!("{}={}", shell_quote(key), shell_quote(value)), + true, + ); + } + + let command_lines = + shell_command_lines(self.command.get_program(), self.command.get_args()); + let last_line = command_lines.len() - 1; + for (index, line) in command_lines.into_iter().enumerate() { + push_shell_line(&mut output, "", &line, index != last_line); + } + + output.pop(); + output + } +} + +fn push_shell_line(output: &mut String, indent: &str, content: &str, continued: bool) { + output.push_str(indent); + output.push_str(content); + if continued { + output.push_str(" \\"); + } + output.push('\n'); +} + +/// Groups option-value pairs so the result reads like a hand-written command. +fn shell_command_lines<'a>( + program: &OsStr, + args: impl Iterator, +) -> Vec { + let args = args.collect::>(); + let mut lines = vec![shell_quote(program)]; + let mut index = 0; + + // Keep command names and leading positional arguments together, such as + // `cargo check` and `cargo test plan_q`. + while index < args.len() && !is_option(args[index]) { + lines[0].push(' '); + lines[0].push_str(&shell_quote(args[index])); + index += 1; + } + + while index < args.len() { + let arg = args[index]; + let mut line = shell_quote(arg); + index += 1; + + if arg == "--" { + while index < args.len() { + line.push(' '); + line.push_str(&shell_quote(args[index])); + index += 1; + } + } else if !arg.to_string_lossy().contains('=') + && index < args.len() + && !is_option(args[index]) + { + line.push(' '); + line.push_str(&shell_quote(args[index])); + index += 1; + } + + lines.push(line); + } + + lines +} + +fn is_option(value: &OsStr) -> bool { + value.to_string_lossy().starts_with('-') +} + +/// Quotes one shell word when it contains characters that need protection. +fn shell_quote(value: &OsStr) -> String { + let value = value.to_string_lossy(); + if !value.is_empty() + && value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'_' | b'@' | b'%' | b'+' | b'=' | b':' | b',' | b'.' | b'/' | b'-' + ) + }) + { + value.into_owned() + } else { + format!("'{}'", value.replace('\'', "'\"'\"'")) + } +} + +fn required_env(name: &str) -> Result { + env::var(name) + .map_err(|_| format!("required environment variable `{name}` is not set")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(args: &[&str]) -> Vec { + args.iter().map(|arg| (*arg).to_string()).collect() + } + + fn command_args(command: &CiCommand) -> Vec { + command + .command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect() + } + + fn context() -> StepContext { + StepContext { + root: PathBuf::from("/workspace"), + } + } + + #[test] + fn check_variants_build_complete_commands() { + let context = context(); + let cases = [ + ( + args(&["workspace"]), + vec![ + "check", + "--profile", + "ci", + "--workspace", + "--all-targets", + "--features", + "integration-tests", + "--locked", + ], + ), + ( + args(&["datafusion", "default"]), + vec![ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion", + ], + ), + ( + args(&["datafusion", "no-default"]), + vec![ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion", + ], + ), + ( + args(&["datafusion", "parquet"]), + vec![ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion", + "--features", + "parquet", + ], + ), + ]; + + for (args, expected) in cases { + let command = context.run_check(&args).unwrap(); + assert_eq!(command_args(&command), expected); + } + } + + #[test] + fn explain_uses_the_same_ci_command() { + let context = context(); + let (action, step, command) = context + .ci_step(&args(&["explain", "test", "benchmark-plan"])) + .unwrap(); + + assert_eq!(action, StepAction::Explain); + assert_eq!(step.command, "test"); + assert_eq!(command.command.get_program(), "cargo"); + assert_eq!( + command.command.get_current_dir(), + Some(Path::new("/workspace")) + ); + assert!(command.full_command().starts_with("cd /workspace && \\\n")); + assert!( + command + .full_command() + .contains("RUST_MIN_STACK=20971520 \\\n") + ); + assert!(command.full_command().contains("cargo test plan_q \\\n")); + assert!(command.full_command().ends_with("-- --test-threads=1")); + } + + #[test] + fn full_command_is_formatted_for_copy_and_paste() { + let command = context() + .run_check(&args(&["datafusion", "default"])) + .unwrap(); + + assert_eq!( + command.full_command(), + concat!( + "cd /workspace && \\\n", + "cargo check \\\n", + "--profile ci \\\n", + "--all-targets \\\n", + "-p datafusion" + ) + ); + assert_eq!(shell_quote(OsStr::new("a b'c")), "'a b'\"'\"'c'"); + } + + #[test] + fn only_check_and_test_are_ci_steps() { + let error = context() + .ci_step(&args(&["fmt"])) + .err() + .expect("fmt must remain outside the CI step interface"); + assert_eq!(error, "unknown CI step `fmt`"); + } + + #[test] + fn help_and_usage_are_generated_from_step_info() { + let help = help(); + assert!(help.starts_with( + "DataFusion CI commands\n\nUsage:\n cargo xtask ci step [args]" + )); + assert!(help.contains(" cargo xtask ci step explain test workspace\n")); + assert!(help.contains("\nAvailable steps:\n")); + assert!(help.contains( + "Shortcut:\n # `cargo ci-step` is short for `cargo xtask ci step`.\n cargo ci-step check workspace\n" + )); + assert!( + help.ends_with("For more details:\n cargo xtask ci step check --help\n") + ); + + for step in CI_STEPS { + assert!(help.contains(step.help_description)); + let details = step_help(step); + assert!(details.contains(&format!("{CI_COMMAND} {}", step.help_usage))); + assert!(details.contains(step.help_description)); + for example in step.help_examples { + assert!(details.contains(&format!("{CI_COMMAND} {example}"))); + } + } + + let error = context() + .ci_step(&args(&["check"])) + .err() + .expect("missing check arguments must show generated usage"); + assert_eq!( + error, + "usage: cargo xtask ci step check [default|no-default|feature]" + ); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs new file mode 100644 index 0000000000000..1614a23fde4b9 --- /dev/null +++ b/xtask/src/main.rs @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! # DataFusion repository development tools +//! +//! ## Example +//! +//! Run the default CI test suite locally: +//! +//! ```sh +//! cargo xtask ci step test workspace +//! ``` +//! +//! Implementing the command inside `xtask` can make it easier to maintain. For example, +//! the local test runs and CI runs can keep in sync. +//! +//! ## Mechanism +//! +//! `xtask` is an internal Rust binary in this workspace. The `cargo xtask` alias +//! builds and runs that binary, forwarding the remaining arguments to it. +//! +//! There is no external dependency required. +//! +//! ## Reference +//! +//! The [`xtask` convention](https://github.com/matklad/cargo-xtask) keeps project +//! automation in ordinary Rust code. It is used by projects such as +//! [rust-analyzer](https://github.com/rust-lang/rust-analyzer/tree/master/xtask). +//! Cargo uses the same pattern for several +//! [`xtask-*` maintenance commands](https://github.com/rust-lang/cargo/blob/master/.cargo/config.toml). +//! +//! # Supported Commands +//! +//! ## CI steps +//! +//! A CI step is one command used by DataFusion's CI that can also be run +//! locally. `explain` prints the complete shell command without executing it. +//! +//! Use `cargo xtask ci step help` to list the available steps and +//! `cargo xtask ci step help` for step-specific arguments. +//! +//! # Examples +//! +//! Show all available CI steps: +//! +//! ```sh +//! cargo xtask help +//! ``` +//! +//! Show the arguments and examples for the `test` step: +//! +//! ```sh +//! cargo xtask ci step test help +//! ``` +//! +//! Run the `cli` variant of the `test` step: +//! +//! ```sh +//! cargo xtask ci step test cli +//! ``` +//! +//! Print that step's shell command without running it: +//! +//! ```sh +//! cargo xtask ci step explain test cli +//! ``` + +mod ci_steps; + +use std::env; +use std::path::PathBuf; + +type Result = std::result::Result; + +fn main() { + let args = env::args().skip(1).collect::>(); + let result = Xtask::new().and_then(|xtask| xtask.run(&args)); + + if let Err(error) = result { + eprintln!("error: {error}"); + std::process::exit(1); + } +} + +struct Xtask { + root: PathBuf, +} + +impl Xtask { + fn new() -> Result { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let root = manifest_dir + .parent() + .ok_or_else(|| { + format!( + "could not find workspace root from {}", + manifest_dir.display() + ) + })? + .to_path_buf(); + Ok(Self { root }) + } + + fn run(&self, args: &[String]) -> Result<()> { + match args { + [] => { + print!("{}", ci_steps::help()); + Ok(()) + } + [help_arg] if ci_steps::is_help_arg(help_arg) => { + print!("{}", ci_steps::help()); + Ok(()) + } + [ci, step, step_args @ ..] if ci == "ci" && step == "step" => { + ci_steps::run(&self.root, step_args) + } + _ => Err(format!("unknown command\n\n{}", ci_steps::help())), + } + } +}