From b162665af9bb01ae85c64d4bf6e005eb0292662d Mon Sep 17 00:00:00 2001 From: Gable Date: Sat, 11 Apr 2026 17:00:20 -0700 Subject: [PATCH 1/3] feat(INFRA-191,192,193): Add shared common.mk, reusable GHA workflows with uv caching, and upgrade pre-commit ruff - Add common.mk with shared Makefile targets (bootstrap, help, activate, deps, lint, format, test/unit, test/unit/coverage, test/unit/fast, docker/*) - Add reusable workflow_call workflows: python-lint.yml and python-unit-tests.yml with enable-cache: true for ~40s speedup via uv caching - Upgrade ruff from v0.7.3 to v0.11.0 in common-checks/.pre-commit-config.yaml --- .github/workflows/python-lint.yml | 69 ++++++++++++ .github/workflows/python-unit-tests.yml | 69 ++++++++++++ common-checks/.pre-commit-config.yaml | 4 +- common.mk | 139 ++++++++++++++++++++++++ 4 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/python-lint.yml create mode 100644 .github/workflows/python-unit-tests.yml create mode 100644 common.mk diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml new file mode 100644 index 0000000..b686530 --- /dev/null +++ b/.github/workflows/python-lint.yml @@ -0,0 +1,69 @@ +name: Python Lint + +on: + workflow_call: + inputs: + python-version: + description: 'Python version to use' + type: string + required: false + default: '3.12' + runner: + description: 'Runner to use' + type: string + required: false + default: 'mdb-dev' + requirements-file: + description: 'Path to requirements file' + type: string + required: false + default: 'requirements/requirements.txt' + has-private-packages: + description: 'Whether private GitHub packages need authentication' + type: boolean + required: false + default: true + make-target: + description: 'Make target to run for linting' + type: string + required: false + default: 'lint' + +jobs: + lint: + runs-on: ${{ inputs.runner }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: ${{ inputs.python-version }} + + - name: Configure GitHub auth for private packages + if: ${{ inputs.has-private-packages }} + env: + GITHUB_TOKEN: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + run: | + printf "machine github.com\nlogin x-access-token\npassword %s\n" "$GITHUB_TOKEN" > ~/.netrc + chmod 600 ~/.netrc + + - name: Install dependencies + run: uv pip install --system --reinstall -r ${{ inputs.requirements-file }} + + - name: Clean up netrc + if: ${{ inputs.has-private-packages }} + run: rm -f ~/.netrc + + - name: Set up CI environment + run: | + if [ -f .env.ci ]; then + cp .env.ci .env + fi + + - name: Run linter + env: + GITHUB_TOKEN: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + run: make ${{ inputs.make-target }} diff --git a/.github/workflows/python-unit-tests.yml b/.github/workflows/python-unit-tests.yml new file mode 100644 index 0000000..d1aa4c1 --- /dev/null +++ b/.github/workflows/python-unit-tests.yml @@ -0,0 +1,69 @@ +name: Python Unit Tests + +on: + workflow_call: + inputs: + python-version: + description: 'Python version to use' + type: string + required: false + default: '3.12' + runner: + description: 'Runner to use' + type: string + required: false + default: 'mdb-dev' + requirements-file: + description: 'Path to requirements file' + type: string + required: false + default: 'requirements/requirements.txt' + has-private-packages: + description: 'Whether private GitHub packages need authentication' + type: boolean + required: false + default: true + make-target: + description: 'Make target to run for tests' + type: string + required: false + default: 'test/unit/coverage' + +jobs: + unit-tests: + runs-on: ${{ inputs.runner }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: ${{ inputs.python-version }} + + - name: Configure GitHub auth for private packages + if: ${{ inputs.has-private-packages }} + env: + GITHUB_TOKEN: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + run: | + printf "machine github.com\nlogin x-access-token\npassword %s\n" "$GITHUB_TOKEN" > ~/.netrc + chmod 600 ~/.netrc + + - name: Install dependencies + run: uv pip install --system --reinstall -r ${{ inputs.requirements-file }} + + - name: Clean up netrc + if: ${{ inputs.has-private-packages }} + run: rm -f ~/.netrc + + - name: Set up CI environment + run: | + if [ -f .env.ci ]; then + cp .env.ci .env + fi + + - name: Run unit tests + env: + GITHUB_TOKEN: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + run: make ${{ inputs.make-target }} diff --git a/common-checks/.pre-commit-config.yaml b/common-checks/.pre-commit-config.yaml index 14d7ce4..f0053f3 100644 --- a/common-checks/.pre-commit-config.yaml +++ b/common-checks/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.3 + rev: v0.11.0 hooks: - id: ruff - id: ruff-format @@ -14,4 +14,4 @@ repos: - id: check-merge-conflict - id: check-added-large-files - id: check-symlinks - - id: check-case-conflict \ No newline at end of file + - id: check-case-conflict diff --git a/common.mk b/common.mk new file mode 100644 index 0000000..3050429 --- /dev/null +++ b/common.mk @@ -0,0 +1,139 @@ +# Common Makefile fragment for MindsDB Python projects +# Include with: -include .make/common.mk +# +# Repos must define these before including: +# MODULE - The Python module name to lint/test (e.g., 'auth', 'minds') +# TEST_PATHS - Path to test directory (e.g., 'tests/unit/') +# COVERAGE_THRESHOLD - Minimum coverage percentage (e.g., '80' or '100') +# +# Optional overrides: +# PYTHON_VERSION - Python version for CI (default: 3.12) +# VENV - Virtual environment path (default: env) +# REQUIREMENTS_FILE - Requirements file for CI (default: requirements/requirements.txt) + +.PHONY: help activate deps lint check/lint format/check format check/fix test/unit test/unit/coverage test/unit/fast docker/build docker/run docker/stop docker/clean bootstrap + +# --- Configuration --- +SHELL := /bin/bash +.ONESHELL: + +# Windows detection +ifeq ($(OS),Windows_NT) + VENV_DIR = Scripts + PYTHON_EXE = python.exe + SET_ENV = set +else + VENV_DIR = bin + PYTHON_EXE = python + SET_ENV = export +endif + +# Defaults (repos can override before including) +PYTHON_VERSION ?= 3.12 +VENV ?= env +REQUIREMENTS_FILE ?= requirements/requirements.txt +IN_CONTAINER := $(shell (test -f /.dockerenv || test -n "$$KUBERNETES_SERVICE_HOST") && echo 1 || echo 0) +DOCKER_COMMAND := $(shell if docker compose version >/dev/null 2>&1; then echo "docker compose"; elif docker-compose version >/dev/null 2>&1; then echo "docker-compose"; fi) + +# Python executable selection +ifeq ($(IN_CONTAINER),1) + PYTHON ?= python +else + PYTHON ?= $(VENV)/$(VENV_DIR)/$(PYTHON_EXE) +endif + +# --- Bootstrap Target --- +bootstrap: ## Clone/pull github-actions repo into .make/ for shared artifacts + @if [ ! -d .make ]; then \ + echo "Creating .make directory..."; \ + mkdir -p .make; \ + fi + @if [ ! -d .make/.git ]; then \ + echo "Cloning github-actions repo..."; \ + git clone --depth 1 https://github.com/mindsdb/github-actions.git .make; \ + else \ + echo "Pulling latest github-actions..."; \ + cd .make && git pull --ff-only && cd ..; \ + fi + @echo "Bootstrap complete. Common artifacts available in .make/" + +# --- Help Target --- +help: ## Display this help message + @echo "Usage: make [target]" + @echo "" + @echo "Available targets:" + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_\/-]+:.*?## / {printf " \033[36m%-25s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +# --- Virtual Environment Targets --- +ifeq ($(IN_CONTAINER),1) +activate: ## No-op inside container (packages already installed) + @echo "Running inside container - using system Python with pre-installed packages." + +clean/venv: ## Remove virtual environment (no-op in container) + @echo "No venv to clean inside container." + +reinstall: ## No-op inside container + @echo "No venv to reinstall inside container." +else +$(VENV)/$(VENV_DIR)/activate: $(REQUIREMENTS_FILE) ## Create virtualenv and install dependencies + @echo "Creating virtual environment at $(VENV)..." + uv venv "$(VENV)" --allow-existing --python $(PYTHON_VERSION) + @echo "Virtual environment created. Installing requirements..." + uv pip install --python "$(VENV)/$(VENV_DIR)/$(PYTHON_EXE)" -r $(REQUIREMENTS_FILE) + @echo "Virtual environment setup complete." + +activate: $(VENV)/$(VENV_DIR)/activate ## Activate the virtual environment + +clean/venv: ## Remove virtual environment + rm -rf $(VENV) + +reinstall: clean/venv activate ## Clean and reinstall everything +endif + +# --- Docker Compose Check --- +deps: ## Check docker compose is available +ifndef DOCKER_COMMAND + @echo "Docker compose not found. Please install either docker-compose (the tool) or the docker compose plugin." + exit 1 +endif + +# --- Linting and Formatting Targets --- +lint: check/lint format/check ## Run all linting and formatting checks + +check/lint: activate ## Check code style with Ruff + $(PYTHON) -m ruff check $(MODULE) + +format/check: activate ## Check code formatting with Ruff + $(PYTHON) -m ruff format $(MODULE) --check + +format: activate ## Format code with Ruff + $(PYTHON) -m ruff format $(MODULE) + +check/fix: activate ## Fix linting issues with Ruff + $(PYTHON) -m ruff check $(MODULE) --fix + +# --- Testing Targets --- +test/unit: activate ## Run unit tests + $(PYTHON) -m pytest $(TEST_PATHS) + +test/unit/coverage: activate ## Run unit tests with coverage + $(PYTHON) -m pytest --cov=$(MODULE) $(TEST_PATHS) --cov-fail-under=$(COVERAGE_THRESHOLD) + +test/unit/fast: activate ## Run only tests affected by changed files (pytest-testmon) + $(PYTHON) -m pytest --testmon $(TEST_PATHS) -p no:randomly --no-header -q + +coverage/html: activate ## Generate HTML coverage report + $(PYTHON) -m pytest --cov=$(MODULE) $(TEST_PATHS) --cov-report html + +# --- Docker Targets --- +docker/build: deps ## Build the docker images + $(SET_ENV) DOCKER_BUILDKIT=1 && $(DOCKER_COMMAND) build + +docker/run: deps ## Run the full application in Docker + $(SET_ENV) DOCKER_BUILDKIT=1 && $(DOCKER_COMMAND) up + +docker/stop: ## Stop the docker containers + $(DOCKER_COMMAND) down + +docker/clean: ## Stop containers and remove volumes + $(DOCKER_COMMAND) down -v From f05b1db2fd9a7baa98cd27636e62054648d5754b Mon Sep 17 00:00:00 2001 From: Gable Date: Sat, 11 Apr 2026 17:26:23 -0700 Subject: [PATCH 2/3] =?UTF-8?q?refactor:=20consistent=20naming=20-=20forma?= =?UTF-8?q?t/check=E2=86=92check/format,=20coverage/html=E2=86=92report/co?= =?UTF-8?q?verage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common.mk | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/common.mk b/common.mk index 3050429..e1728f4 100644 --- a/common.mk +++ b/common.mk @@ -11,7 +11,7 @@ # VENV - Virtual environment path (default: env) # REQUIREMENTS_FILE - Requirements file for CI (default: requirements/requirements.txt) -.PHONY: help activate deps lint check/lint format/check format check/fix test/unit test/unit/coverage test/unit/fast docker/build docker/run docker/stop docker/clean bootstrap +.PHONY: help activate deps lint check/lint check/format format check/fix test/unit test/unit/coverage test/unit/fast test/report docker/build docker/run docker/stop docker/clean # --- Configuration --- SHELL := /bin/bash @@ -42,21 +42,6 @@ else PYTHON ?= $(VENV)/$(VENV_DIR)/$(PYTHON_EXE) endif -# --- Bootstrap Target --- -bootstrap: ## Clone/pull github-actions repo into .make/ for shared artifacts - @if [ ! -d .make ]; then \ - echo "Creating .make directory..."; \ - mkdir -p .make; \ - fi - @if [ ! -d .make/.git ]; then \ - echo "Cloning github-actions repo..."; \ - git clone --depth 1 https://github.com/mindsdb/github-actions.git .make; \ - else \ - echo "Pulling latest github-actions..."; \ - cd .make && git pull --ff-only && cd ..; \ - fi - @echo "Bootstrap complete. Common artifacts available in .make/" - # --- Help Target --- help: ## Display this help message @echo "Usage: make [target]" @@ -98,12 +83,12 @@ ifndef DOCKER_COMMAND endif # --- Linting and Formatting Targets --- -lint: check/lint format/check ## Run all linting and formatting checks +lint: check/lint check/format ## Run all linting and formatting checks check/lint: activate ## Check code style with Ruff $(PYTHON) -m ruff check $(MODULE) -format/check: activate ## Check code formatting with Ruff +check/format: activate ## Check code formatting with Ruff $(PYTHON) -m ruff format $(MODULE) --check format: activate ## Format code with Ruff @@ -122,7 +107,7 @@ test/unit/coverage: activate ## Run unit tests with coverage test/unit/fast: activate ## Run only tests affected by changed files (pytest-testmon) $(PYTHON) -m pytest --testmon $(TEST_PATHS) -p no:randomly --no-header -q -coverage/html: activate ## Generate HTML coverage report +test/report: activate ## Generate HTML coverage report $(PYTHON) -m pytest --cov=$(MODULE) $(TEST_PATHS) --cov-report html # --- Docker Targets --- From 19b087ea48eb46a8fd9147edb553e1076014e694 Mon Sep 17 00:00:00 2001 From: Gable Date: Sat, 11 Apr 2026 18:37:43 -0700 Subject: [PATCH 3/3] I broke this --- .github/workflows/python-lint.yml | 20 ++++++++++---------- .github/workflows/python-unit-tests.yml | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index b686530..0e46a46 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -4,30 +4,30 @@ on: workflow_call: inputs: python-version: - description: 'Python version to use' + description: "Python version to use" type: string required: false - default: '3.12' + default: "3.12" runner: - description: 'Runner to use' + description: "Runner to use" type: string required: false - default: 'mdb-dev' + default: "mdb-dev" requirements-file: - description: 'Path to requirements file' + description: "Path to requirements file" type: string required: false - default: 'requirements/requirements.txt' + default: "requirements/requirements.txt" has-private-packages: - description: 'Whether private GitHub packages need authentication' + description: "Whether private GitHub packages need authentication" type: boolean required: false default: true make-target: - description: 'Make target to run for linting' + description: "Make target to run for linting" type: string required: false - default: 'lint' + default: "lint" jobs: lint: @@ -51,7 +51,7 @@ jobs: chmod 600 ~/.netrc - name: Install dependencies - run: uv pip install --system --reinstall -r ${{ inputs.requirements-file }} + run: uv pip install --system -r ${{ inputs.requirements-file }} - name: Clean up netrc if: ${{ inputs.has-private-packages }} diff --git a/.github/workflows/python-unit-tests.yml b/.github/workflows/python-unit-tests.yml index d1aa4c1..00a6d08 100644 --- a/.github/workflows/python-unit-tests.yml +++ b/.github/workflows/python-unit-tests.yml @@ -4,30 +4,30 @@ on: workflow_call: inputs: python-version: - description: 'Python version to use' + description: "Python version to use" type: string required: false - default: '3.12' + default: "3.12" runner: - description: 'Runner to use' + description: "Runner to use" type: string required: false - default: 'mdb-dev' + default: "mdb-dev" requirements-file: - description: 'Path to requirements file' + description: "Path to requirements file" type: string required: false - default: 'requirements/requirements.txt' + default: "requirements/requirements.txt" has-private-packages: - description: 'Whether private GitHub packages need authentication' + description: "Whether private GitHub packages need authentication" type: boolean required: false default: true make-target: - description: 'Make target to run for tests' + description: "Make target to run for tests" type: string required: false - default: 'test/unit/coverage' + default: "test/unit/coverage" jobs: unit-tests: @@ -51,7 +51,7 @@ jobs: chmod 600 ~/.netrc - name: Install dependencies - run: uv pip install --system --reinstall -r ${{ inputs.requirements-file }} + run: uv pip install --system -r ${{ inputs.requirements-file }} - name: Clean up netrc if: ${{ inputs.has-private-packages }}