diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml new file mode 100644 index 0000000..0e46a46 --- /dev/null +++ b/.github/workflows/python-lint.yml @@ -0,0 +1,69 @@ +name: Python Lint + +on: + workflow_call: + inputs: + python-version: + description: "Python version to use" + type: string + required: false + default: "3.12" + runner: + description: "Runner to use" + type: string + required: false + default: "mdb-dev" + requirements-file: + description: "Path to requirements file" + type: string + required: false + default: "requirements/requirements.txt" + has-private-packages: + description: "Whether private GitHub packages need authentication" + type: boolean + required: false + default: true + make-target: + description: "Make target to run for linting" + type: string + required: false + default: "lint" + +jobs: + lint: + runs-on: ${{ inputs.runner }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: ${{ inputs.python-version }} + + - name: Configure GitHub auth for private packages + if: ${{ inputs.has-private-packages }} + env: + GITHUB_TOKEN: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + run: | + printf "machine github.com\nlogin x-access-token\npassword %s\n" "$GITHUB_TOKEN" > ~/.netrc + chmod 600 ~/.netrc + + - name: Install dependencies + run: uv pip install --system -r ${{ inputs.requirements-file }} + + - name: Clean up netrc + if: ${{ inputs.has-private-packages }} + run: rm -f ~/.netrc + + - name: Set up CI environment + run: | + if [ -f .env.ci ]; then + cp .env.ci .env + fi + + - name: Run linter + env: + GITHUB_TOKEN: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + run: make ${{ inputs.make-target }} diff --git a/.github/workflows/python-unit-tests.yml b/.github/workflows/python-unit-tests.yml new file mode 100644 index 0000000..00a6d08 --- /dev/null +++ b/.github/workflows/python-unit-tests.yml @@ -0,0 +1,69 @@ +name: Python Unit Tests + +on: + workflow_call: + inputs: + python-version: + description: "Python version to use" + type: string + required: false + default: "3.12" + runner: + description: "Runner to use" + type: string + required: false + default: "mdb-dev" + requirements-file: + description: "Path to requirements file" + type: string + required: false + default: "requirements/requirements.txt" + has-private-packages: + description: "Whether private GitHub packages need authentication" + type: boolean + required: false + default: true + make-target: + description: "Make target to run for tests" + type: string + required: false + default: "test/unit/coverage" + +jobs: + unit-tests: + runs-on: ${{ inputs.runner }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: ${{ inputs.python-version }} + + - name: Configure GitHub auth for private packages + if: ${{ inputs.has-private-packages }} + env: + GITHUB_TOKEN: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + run: | + printf "machine github.com\nlogin x-access-token\npassword %s\n" "$GITHUB_TOKEN" > ~/.netrc + chmod 600 ~/.netrc + + - name: Install dependencies + run: uv pip install --system -r ${{ inputs.requirements-file }} + + - name: Clean up netrc + if: ${{ inputs.has-private-packages }} + run: rm -f ~/.netrc + + - name: Set up CI environment + run: | + if [ -f .env.ci ]; then + cp .env.ci .env + fi + + - name: Run unit tests + env: + GITHUB_TOKEN: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + run: make ${{ inputs.make-target }} diff --git a/common-checks/.pre-commit-config.yaml b/common-checks/.pre-commit-config.yaml index 14d7ce4..f0053f3 100644 --- a/common-checks/.pre-commit-config.yaml +++ b/common-checks/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.3 + rev: v0.11.0 hooks: - id: ruff - id: ruff-format @@ -14,4 +14,4 @@ repos: - id: check-merge-conflict - id: check-added-large-files - id: check-symlinks - - id: check-case-conflict \ No newline at end of file + - id: check-case-conflict diff --git a/common.mk b/common.mk new file mode 100644 index 0000000..e1728f4 --- /dev/null +++ b/common.mk @@ -0,0 +1,124 @@ +# Common Makefile fragment for MindsDB Python projects +# Include with: -include .make/common.mk +# +# Repos must define these before including: +# MODULE - The Python module name to lint/test (e.g., 'auth', 'minds') +# TEST_PATHS - Path to test directory (e.g., 'tests/unit/') +# COVERAGE_THRESHOLD - Minimum coverage percentage (e.g., '80' or '100') +# +# Optional overrides: +# PYTHON_VERSION - Python version for CI (default: 3.12) +# VENV - Virtual environment path (default: env) +# REQUIREMENTS_FILE - Requirements file for CI (default: requirements/requirements.txt) + +.PHONY: help activate deps lint check/lint check/format format check/fix test/unit test/unit/coverage test/unit/fast test/report docker/build docker/run docker/stop docker/clean + +# --- Configuration --- +SHELL := /bin/bash +.ONESHELL: + +# Windows detection +ifeq ($(OS),Windows_NT) + VENV_DIR = Scripts + PYTHON_EXE = python.exe + SET_ENV = set +else + VENV_DIR = bin + PYTHON_EXE = python + SET_ENV = export +endif + +# Defaults (repos can override before including) +PYTHON_VERSION ?= 3.12 +VENV ?= env +REQUIREMENTS_FILE ?= requirements/requirements.txt +IN_CONTAINER := $(shell (test -f /.dockerenv || test -n "$$KUBERNETES_SERVICE_HOST") && echo 1 || echo 0) +DOCKER_COMMAND := $(shell if docker compose version >/dev/null 2>&1; then echo "docker compose"; elif docker-compose version >/dev/null 2>&1; then echo "docker-compose"; fi) + +# Python executable selection +ifeq ($(IN_CONTAINER),1) + PYTHON ?= python +else + PYTHON ?= $(VENV)/$(VENV_DIR)/$(PYTHON_EXE) +endif + +# --- Help Target --- +help: ## Display this help message + @echo "Usage: make [target]" + @echo "" + @echo "Available targets:" + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_\/-]+:.*?## / {printf " \033[36m%-25s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +# --- Virtual Environment Targets --- +ifeq ($(IN_CONTAINER),1) +activate: ## No-op inside container (packages already installed) + @echo "Running inside container - using system Python with pre-installed packages." + +clean/venv: ## Remove virtual environment (no-op in container) + @echo "No venv to clean inside container." + +reinstall: ## No-op inside container + @echo "No venv to reinstall inside container." +else +$(VENV)/$(VENV_DIR)/activate: $(REQUIREMENTS_FILE) ## Create virtualenv and install dependencies + @echo "Creating virtual environment at $(VENV)..." + uv venv "$(VENV)" --allow-existing --python $(PYTHON_VERSION) + @echo "Virtual environment created. Installing requirements..." + uv pip install --python "$(VENV)/$(VENV_DIR)/$(PYTHON_EXE)" -r $(REQUIREMENTS_FILE) + @echo "Virtual environment setup complete." + +activate: $(VENV)/$(VENV_DIR)/activate ## Activate the virtual environment + +clean/venv: ## Remove virtual environment + rm -rf $(VENV) + +reinstall: clean/venv activate ## Clean and reinstall everything +endif + +# --- Docker Compose Check --- +deps: ## Check docker compose is available +ifndef DOCKER_COMMAND + @echo "Docker compose not found. Please install either docker-compose (the tool) or the docker compose plugin." + exit 1 +endif + +# --- Linting and Formatting Targets --- +lint: check/lint check/format ## Run all linting and formatting checks + +check/lint: activate ## Check code style with Ruff + $(PYTHON) -m ruff check $(MODULE) + +check/format: activate ## Check code formatting with Ruff + $(PYTHON) -m ruff format $(MODULE) --check + +format: activate ## Format code with Ruff + $(PYTHON) -m ruff format $(MODULE) + +check/fix: activate ## Fix linting issues with Ruff + $(PYTHON) -m ruff check $(MODULE) --fix + +# --- Testing Targets --- +test/unit: activate ## Run unit tests + $(PYTHON) -m pytest $(TEST_PATHS) + +test/unit/coverage: activate ## Run unit tests with coverage + $(PYTHON) -m pytest --cov=$(MODULE) $(TEST_PATHS) --cov-fail-under=$(COVERAGE_THRESHOLD) + +test/unit/fast: activate ## Run only tests affected by changed files (pytest-testmon) + $(PYTHON) -m pytest --testmon $(TEST_PATHS) -p no:randomly --no-header -q + +test/report: activate ## Generate HTML coverage report + $(PYTHON) -m pytest --cov=$(MODULE) $(TEST_PATHS) --cov-report html + +# --- Docker Targets --- +docker/build: deps ## Build the docker images + $(SET_ENV) DOCKER_BUILDKIT=1 && $(DOCKER_COMMAND) build + +docker/run: deps ## Run the full application in Docker + $(SET_ENV) DOCKER_BUILDKIT=1 && $(DOCKER_COMMAND) up + +docker/stop: ## Stop the docker containers + $(DOCKER_COMMAND) down + +docker/clean: ## Stop containers and remove volumes + $(DOCKER_COMMAND) down -v