From 08b47db13fafa8b77bc3707ab857328e13b6063c Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Tue, 13 Jan 2026 08:35:28 +0000 Subject: [PATCH 1/5] migrate to Checks API for integration tests Use GitHub Checks API instead of Statuses API to report integration test results. This enables the use of GitHub App authentication and eliminates the need for monthly PAT rotation. Changes: - Generate a second GitHub App token for check creation - Create check run before triggering tests in eng-dev-ecosystem - Pass check_run_id to the workflow for status updates - Update get_status() to query Checks API instead of Statuses API --- .github/workflows/start-integration-tests.yml | 11 ++- tools/start_integration_tests.py | 92 ++++++++++++++----- 2 files changed, 78 insertions(+), 25 deletions(-) diff --git a/.github/workflows/start-integration-tests.yml b/.github/workflows/start-integration-tests.yml index 66ecfde2d4..ce3649a25e 100644 --- a/.github/workflows/start-integration-tests.yml +++ b/.github/workflows/start-integration-tests.yml @@ -22,7 +22,7 @@ jobs: if: "${{ !github.event.pull_request.head.repo.fork }}" steps: - - name: Generate GitHub App Token + - name: Generate GitHub App Token for Workflow Trigger id: generate-token uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 with: @@ -31,11 +31,20 @@ jobs: owner: ${{ secrets.ORG_NAME }} repositories: ${{secrets.REPO_NAME}} + - name: Generate GitHub App Token for Check Updates + id: generate-check-token + uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 + with: + app-id: ${{ secrets.DECO_TEST_APPROVAL_APP_ID }} + private-key: ${{ secrets.DECO_TEST_APPROVAL_PRIVATE_KEY }} + owner: databricks + - name: Fetch start_integration_tests.py run: wget https://raw.githubusercontent.com/databricks/cli/refs/heads/main/tools/start_integration_tests.py - name: Run start_integration_tests.py env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} + GH_CHECK_TOKEN: ${{ steps.generate-check-token.outputs.token }} run: |- python3 ./start_integration_tests.py -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} --yes diff --git a/tools/start_integration_tests.py b/tools/start_integration_tests.py index 088c2a1711..8d353b9d38 100755 --- a/tools/start_integration_tests.py +++ b/tools/start_integration_tests.py @@ -8,10 +8,9 @@ import argparse import json +import os import subprocess import sys -from pathlib import Path -import re CLI_REPO = "databricks/cli" @@ -20,14 +19,17 @@ ALLOWED_HEAD_OWNER = {"id": "MDEyOk9yZ2FuaXphdGlvbjQ5OTgwNTI=", "login": "databricks"} -def run(cmd): +def run(cmd, env=None): sys.stderr.write("+ " + " ".join(cmd) + "\n") - return subprocess.run(cmd, check=True) + return subprocess.run(cmd, check=True, env=env) -def run_json(cmd): +def run_json(cmd, env=None): sys.stderr.write("+ " + " ".join(cmd) + "\n") - result = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf-8", check=True) + run_env = os.environ.copy() + if env: + run_env.update(env) + result = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf-8", check=True, env=run_env) try: return json.loads(result.stdout) @@ -36,6 +38,38 @@ def run_json(cmd): raise +def create_check(commit_sha): + """Create a check run for the given commit and return the check_run_id.""" + check_token = os.environ.get("GH_CHECK_TOKEN") + if not check_token: + print("Warning: GH_CHECK_TOKEN not set, skipping check creation") + return None + + response = run_json( + [ + "gh", + "api", + "-X", + "POST", + f"/repos/{CLI_REPO}/check-runs", + "-f", + "name=Integration Tests", + "-f", + f"head_sha={commit_sha}", + "-f", + "status=queued", + "-f", + "output[title]=Integration Tests", + "-f", + "output[summary]=Tests queued and will be triggered shortly...", + ], + env={"GH_TOKEN": check_token}, + ) + check_run_id = response.get("id") + print(f"Created check run: {check_run_id}") + return check_run_id + + def get_approved_prs_by_non_team(): prs = run_json( [ @@ -108,30 +142,40 @@ def start_job(pr_number, commit_sha, author, approved_by, workflow, repo, force= response = input("Start integration tests? (y/n): ") if response.lower() == "y": - result = run( - [ - "gh", - "workflow", - "run", - workflow, - "-R", - repo, - "-F", - f"pull_request_number={pr_number}", - "-F", - f"commit_sha={commit_sha}", - ], - ) + check_run_id = create_check(commit_sha) + + cmd = [ + "gh", + "workflow", + "run", + workflow, + "-R", + repo, + "-F", + f"pull_request_number={pr_number}", + "-F", + f"commit_sha={commit_sha}", + ] + if check_run_id: + cmd.extend(["-F", f"check_run_id={check_run_id}"]) + + run(cmd) print(f"Started integration tests for PR #{pr_number}") def get_status(commit_sha): - statuses = run_json(["gh", "api", f"repos/{CLI_REPO}/commits/{commit_sha}/statuses"]) + response = run_json(["gh", "api", f"repos/{CLI_REPO}/commits/{commit_sha}/check-runs"]) result = [] - for st in statuses: - if st["context"] != "Integration Tests Check": + for check in response.get("check_runs", []): + if check["name"] != "Integration Tests": continue - result.append(f"{st['state']} {st['target_url']}") + status = check["status"] + conclusion = check.get("conclusion", "") + details_url = check.get("details_url", "") + if conclusion: + result.append(f"{conclusion} {details_url}") + else: + result.append(f"{status} {details_url}") return result From f78191e83a256cfdc595b7cac39abcbbcb508cb6 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 19 Jan 2026 11:28:07 +0000 Subject: [PATCH 2/5] add --summary-file flag to gh_report and gh_parse allows ecosystem workflows to extract just the summary line without grep parsing --- tools/gh_parse.py | 17 ++++++++++++++++- tools/gh_report.py | 3 +++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/gh_parse.py b/tools/gh_parse.py index 3425aa3e79..2a57c16c80 100755 --- a/tools/gh_parse.py +++ b/tools/gh_parse.py @@ -353,7 +353,7 @@ def mark_known_failures(results, known_failures_config): return marked_results -def print_report(filenames, filter, filter_env, show_output, markdown=False, omit_repl=False): +def print_report(filenames, filter, filter_env, show_output, markdown=False, omit_repl=False, summary_file=None): known_failures_config = load_known_failures() outputs = {} # test_key -> env -> [output] per_test_per_env_stats = {} # test_key -> env -> action -> count @@ -536,12 +536,25 @@ def key(column): } ) table_txt = format_table(table, markdown=markdown) + summary_msg = None if len(table) > 5: summary_msg = make_summary_message(table, summary) table_txt = wrap_in_details(table_txt, summary_msg) if table_txt: print(table_txt) + # Write summary to file if requested + if summary_file: + if summary_msg: + Path(summary_file).write_text(summary_msg) + elif len(table) > 0: + # If we have tests but not enough for full summary, create a simple one + summary_msg = make_summary_message(table, summary) + Path(summary_file).write_text(summary_msg) + else: + # No interesting tests found + Path(summary_file).write_text("All tests passed") + # Generate slowest tests table (tests slower than 10 minutes) all_durations = [] # [(env, package, testname, duration), ...] for env, env_durations in durations_by_env.items(): @@ -691,6 +704,7 @@ def main(): help="Omit lines starting with 'REPL' and containing 'Available replacements:'", action="store_true", ) + parser.add_argument("--summary-file", help="Write summary line to this file") args = parser.parse_args() print_report( args.filenames, @@ -699,6 +713,7 @@ def main(): show_output=args.output, markdown=args.markdown, omit_repl=args.omit_repl, + summary_file=args.summary_file, ) diff --git a/tools/gh_report.py b/tools/gh_report.py index ff1b69f416..21c0bea4ca 100755 --- a/tools/gh_report.py +++ b/tools/gh_report.py @@ -175,6 +175,7 @@ def main(): help="Omit lines starting with 'REPL' and containing 'Available replacements:'", action="store_true", ) + parser.add_argument("--summary-file", help="Write summary line to this file") # This does not work because we don't store artifacts for unit tests. We could download logs instead but that requires different parsing method: # ~/work/cli % gh api -H "Accept: application/vnd.github+json" /repos/databricks/cli/actions/runs/15827411452/logs > logs.zip @@ -217,6 +218,8 @@ def main(): cmd.append("--markdown") if args.omit_repl: cmd.append("--omit-repl") + if args.summary_file: + cmd.extend(["--summary-file", args.summary_file]) cmd.append(f"{target_dir}") run(cmd, shell=True) From 75b2e2e4f6dea752be85c2ce678e32a0077ac6d6 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 19 Jan 2026 11:34:52 +0000 Subject: [PATCH 3/5] temp: point to ecosystem test branch for testing temporarily use omer-lachish_data/deco-26173-cli-report-in-checks branch to test new check report functionality before merging --- tools/start_integration_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/start_integration_tests.py b/tools/start_integration_tests.py index 8d353b9d38..d8c8734aa5 100755 --- a/tools/start_integration_tests.py +++ b/tools/start_integration_tests.py @@ -151,6 +151,8 @@ def start_job(pr_number, commit_sha, author, approved_by, workflow, repo, force= workflow, "-R", repo, + "--ref", + "omer-lachish_data/deco-26173-cli-report-in-checks", "-F", f"pull_request_number={pr_number}", "-F", From 5271caa6c66b4dc9945e95607d15220a416ef778 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 19 Jan 2026 15:24:24 +0000 Subject: [PATCH 4/5] remove temporary testing artifacts --- .github/workflows/start-integration-tests.yml | 2 -- tools/start_integration_tests.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/start-integration-tests.yml b/.github/workflows/start-integration-tests.yml index ce3649a25e..94f5b88586 100644 --- a/.github/workflows/start-integration-tests.yml +++ b/.github/workflows/start-integration-tests.yml @@ -1,8 +1,6 @@ name: start-integration-tests on: - #schedule: - # - cron: '*/10 * * * *' workflow_dispatch: jobs: diff --git a/tools/start_integration_tests.py b/tools/start_integration_tests.py index d8c8734aa5..0f9b7a77e3 100755 --- a/tools/start_integration_tests.py +++ b/tools/start_integration_tests.py @@ -152,7 +152,7 @@ def start_job(pr_number, commit_sha, author, approved_by, workflow, repo, force= "-R", repo, "--ref", - "omer-lachish_data/deco-26173-cli-report-in-checks", + "main", "-F", f"pull_request_number={pr_number}", "-F", From 02642c5f192ee900c7148d1a206dd4d6662ad34e Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 19 Jan 2026 15:51:13 +0000 Subject: [PATCH 5/5] remove --summary-file changes (will be in separate PR) --- tools/gh_parse.py | 17 +---------------- tools/gh_report.py | 3 --- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/tools/gh_parse.py b/tools/gh_parse.py index 2a57c16c80..3425aa3e79 100755 --- a/tools/gh_parse.py +++ b/tools/gh_parse.py @@ -353,7 +353,7 @@ def mark_known_failures(results, known_failures_config): return marked_results -def print_report(filenames, filter, filter_env, show_output, markdown=False, omit_repl=False, summary_file=None): +def print_report(filenames, filter, filter_env, show_output, markdown=False, omit_repl=False): known_failures_config = load_known_failures() outputs = {} # test_key -> env -> [output] per_test_per_env_stats = {} # test_key -> env -> action -> count @@ -536,25 +536,12 @@ def key(column): } ) table_txt = format_table(table, markdown=markdown) - summary_msg = None if len(table) > 5: summary_msg = make_summary_message(table, summary) table_txt = wrap_in_details(table_txt, summary_msg) if table_txt: print(table_txt) - # Write summary to file if requested - if summary_file: - if summary_msg: - Path(summary_file).write_text(summary_msg) - elif len(table) > 0: - # If we have tests but not enough for full summary, create a simple one - summary_msg = make_summary_message(table, summary) - Path(summary_file).write_text(summary_msg) - else: - # No interesting tests found - Path(summary_file).write_text("All tests passed") - # Generate slowest tests table (tests slower than 10 minutes) all_durations = [] # [(env, package, testname, duration), ...] for env, env_durations in durations_by_env.items(): @@ -704,7 +691,6 @@ def main(): help="Omit lines starting with 'REPL' and containing 'Available replacements:'", action="store_true", ) - parser.add_argument("--summary-file", help="Write summary line to this file") args = parser.parse_args() print_report( args.filenames, @@ -713,7 +699,6 @@ def main(): show_output=args.output, markdown=args.markdown, omit_repl=args.omit_repl, - summary_file=args.summary_file, ) diff --git a/tools/gh_report.py b/tools/gh_report.py index 21c0bea4ca..ff1b69f416 100755 --- a/tools/gh_report.py +++ b/tools/gh_report.py @@ -175,7 +175,6 @@ def main(): help="Omit lines starting with 'REPL' and containing 'Available replacements:'", action="store_true", ) - parser.add_argument("--summary-file", help="Write summary line to this file") # This does not work because we don't store artifacts for unit tests. We could download logs instead but that requires different parsing method: # ~/work/cli % gh api -H "Accept: application/vnd.github+json" /repos/databricks/cli/actions/runs/15827411452/logs > logs.zip @@ -218,8 +217,6 @@ def main(): cmd.append("--markdown") if args.omit_repl: cmd.append("--omit-repl") - if args.summary_file: - cmd.extend(["--summary-file", args.summary_file]) cmd.append(f"{target_dir}") run(cmd, shell=True)