From 7247b2facd5003b015477bd9e98f370b5e780011 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Wed, 6 May 2026 11:50:30 +0100 Subject: [PATCH 1/3] Replaced and restructured GitHub Actions workflows to streamline testing, staging, and production deployments. --- .github/ISSUE_TEMPLATE/release-request.yml | 53 ++++++ .github/workflows/pre_production_tests.yml | 49 ++++++ .github/workflows/production-deploy.yml | 62 +++++++ .github/workflows/staging-deploy.yml | 74 ++++++++ .github/workflows/test-and-deploy.yml | 193 --------------------- .github/workflows/test-lint.yml | 55 ++++++ README.md | 4 + 7 files changed, 297 insertions(+), 193 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/release-request.yml create mode 100644 .github/workflows/pre_production_tests.yml create mode 100644 .github/workflows/production-deploy.yml create mode 100644 .github/workflows/staging-deploy.yml delete mode 100755 .github/workflows/test-and-deploy.yml create mode 100644 .github/workflows/test-lint.yml diff --git a/.github/ISSUE_TEMPLATE/release-request.yml b/.github/ISSUE_TEMPLATE/release-request.yml new file mode 100644 index 0000000..2ed9717 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/release-request.yml @@ -0,0 +1,53 @@ +name: "Release Request" +description: "Request a deployment by specifying the evaluation function, changes, target commit/branch, and test confirmation." +title: "Release Request" +labels: + - release + - deployment +assignees: [] +body: + - type: textarea + id: description_of_changes + attributes: + label: Description of changes + description: "Summarize what is changing and why. Include links to PRs, issues, or changelogs." + placeholder: | + - What changed: + - Why: + - Related PRs/Issues: #123, #456 + render: markdown + validations: + required: true + + - type: input + id: branch_to_deploy + attributes: + label: Branch to deploy + description: | + Specify Branch name to deploy. + placeholder: "e.g., release/2025-09-29" + validations: + required: true + + - type: dropdown + id: version-bump + attributes: + label: "๐Ÿš€ What kind of update is this?" + description: "Tell us how significant this change is. This helps us set the correct new version number." + options: + - "Patch: A small fix for a bug. It won't break anything for existing users. (e.g., 1.2.3 โž” 1.2.4)" + - "Minor: Adds a new feature, but doesn't change how existing ones work. A safe update. (e.g., 1.2.3 โž” 1.3.0)" + - "Major: A big change that alters existing features. Users may need to update their work to adapt. (e.g., 1.2.3 โž” 2.0.0)" + default: 0 + validations: + required: true + + - type: markdown + attributes: + value: | + --- + ### โšก Click the Link Below to Run the Workflow + + Clicking the link will take you to the Actions page. You will need to click the **"Run workflow"** button there to start the process. + + ## [โžก๏ธ Go to Workflow Run Page](../actions/workflows/production-deploy.yml) \ No newline at end of file diff --git a/.github/workflows/pre_production_tests.yml b/.github/workflows/pre_production_tests.yml new file mode 100644 index 0000000..2daabd1 --- /dev/null +++ b/.github/workflows/pre_production_tests.yml @@ -0,0 +1,49 @@ +name: Run Pre-Production Validation Tests + +on: + workflow_dispatch: + inputs: + eval_function: + type: string + description: "The name of the evaluation function to test" + required: true + sql_limit: + type: number + description: "The maximum number of SQL test cases to run" + required: false + default: 500 + seed: + type: string + description: "Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate." + required: false + default: '' + request_delay: + type: string + description: "Delay (seconds) between dispatching requests" + required: false + default: '0' + max_concurrency: + type: string + description: "Max concurrent requests (lower for GPT-backed functions)" + required: false + default: '5' + +jobs: + run-pre-production-tests: + name: ๐Ÿงช Run Staging Validation Tests + uses: lambda-feedback/Database-Testing/.github/workflows/test_evaluation_function.yml@main + with: + eval_function: ${{ inputs.eval_function }} + sql_limit: ${{ inputs.sql_limit }} + seed: ${{ inputs.seed }} + request_delay: ${{ inputs.request_delay }} + max_concurrency: ${{ inputs.max_concurrency }} + secrets: + TEST_API_ENDPOINT: ${{ secrets.TEST_API_ENDPOINT }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_HOST: ${{ secrets.DB_HOST }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_NAME: ${{ secrets.DB_NAME }} + GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_DB_CREDS }} + GCP_PROJECT_ID: ${{ secrets.GCP_DB_PROJECT_ID }} \ No newline at end of file diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml new file mode 100644 index 0000000..ec071c3 --- /dev/null +++ b/.github/workflows/production-deploy.yml @@ -0,0 +1,62 @@ +name: Deploy to Production + +on: + workflow_dispatch: + inputs: + version-bump: + description: 'Version bump type' + required: true + type: choice + options: + - patch + - minor + - major + default: patch + branch: + description: 'Branch to release from' + required: true + type: string + default: 'main' + seed: + description: 'Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate.' + required: false + type: string + default: '' + request_delay: + description: 'Delay (seconds) between dispatching requests' + required: false + type: string + default: '0' + max_concurrency: + description: 'Max concurrent requests (lower for GPT-backed functions)' + required: false + type: string + default: '5' +jobs: + deploy: + uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@deploy-request + with: + template-repository-name: 'lambda-feedback/evaluation-function-boilerplate-python' + build-file: "app/Dockerfile" + build-context: "./app" + environment: "production" + version-bump: ${{ inputs.version-bump }} + branch: ${{ inputs.branch }} + run-database-tests: true + seed: ${{ inputs.seed }} + request_delay: ${{ inputs.request_delay }} + max_concurrency: ${{ inputs.max_concurrency }} + + secrets: + aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} + aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} + function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY}} + gcp_credentials: ${{ secrets.GCP_DEPLOY }} + TEST_API_ENDPOINT: ${{ secrets.TEST_API_ENDPOINT }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_HOST: ${{ secrets.DB_HOST }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_NAME: ${{ secrets.DB_NAME }} + GCP_DB_CREDS: ${{ secrets.GCP_DB_CREDS }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} \ No newline at end of file diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml new file mode 100644 index 0000000..b03a9ec --- /dev/null +++ b/.github/workflows/staging-deploy.yml @@ -0,0 +1,74 @@ +name: Deploy to Staging + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + checks: write + pull-requests: write + strategy: + fail-fast: false + matrix: + python-version: ["3.8"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m venv .venv + .venv/bin/pip install pytest + .venv/bin/pip install -r app/requirements.txt + + + - name: Lint with flake8 + run: | + source .venv/bin/activate + .venv/bin/pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 ./app --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./app --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Run tests + if: always() + run: | + source .venv/bin/activate + pytest --junit-xml=./reports/pytest.xml --tb=auto -v + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results-${{ matrix.python-version }} + path: ./reports/pytest.xml + if-no-files-found: warn + deploy: + needs: test + uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@deploy-request + with: + template-repository-name: "lambda-feedback/evaluation-function-boilerplate-python" + build-file: "app/Dockerfile" + build-context: "./app" + build-platforms: "aws" + environment: "staging" + lfs: false + secrets: + aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} + aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} + function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY}} + gcp_credentials: ${{ secrets.GCP_DEPLOY }} diff --git a/.github/workflows/test-and-deploy.yml b/.github/workflows/test-and-deploy.yml deleted file mode 100755 index bbd6a9e..0000000 --- a/.github/workflows/test-and-deploy.yml +++ /dev/null @@ -1,193 +0,0 @@ -name: Test & Deploy Evaluation Function to AWS Lambda - -on: - push: - branches: - - master - - main - workflow_dispatch: - -jobs: - test: - name: Test - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: [3.8] - - defaults: - run: - working-directory: app/ - - env: - SCHEMAS_URL: https://raw.githubusercontent.com/lambda-feedback/request-response-schemas/master/ - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - python -m pip install -r requirements.txt - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Test Evaluation Function - run: | - pytest -v evaluation_tests.py::TestEvaluationFunction - - - name: Test Preview Function - run: | - pytest -v preview_tests.py::TestPreviewFunction - - deploy-staging: - name: Deploy Staging - needs: test - runs-on: ubuntu-latest - environment: production - env: - ECR_REPOSITORY: lambda-feedback-staging-functions-repository - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set config.json output - id: set_config_var - run: | - content=`cat ./config.json` - # the following lines are only required for multi line json - content="${content//'%'/'%25'}" - content="${content//$'\n'/'%0A'}" - content="${content//$'\r'/'%0D'}" - # end of optional handling for multi line json - echo "::set-output name=configJson::$content" - - - name: set Evaluation Function Name - id: set_function_name - run: | - functionName="${{fromJson(steps.set_config_var.outputs.configJson).EvaluationFunctionName}}" - [[ -z "$functionName" ]] && { echo "Add EvaluationFunctionName to config.json" ; exit 1; } - echo "::set-output name=function_name::$functionName" - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} - aws-secret-access-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET }} - aws-region: eu-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - # Build docker image from algorithm, schema and requirements - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG app/ - # Push image to ECR - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" - - - name: deploy evaluation function - id: deploy-evaluation-function - env: - BACKEND_API_URL: https://staging-api.lambdafeedback.com - API_KEY: ${{ secrets.FUNCTION_ADMIN_API_KEY }} - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - curl --location --request POST "$BACKEND_API_URL/grading-function/ensure" \ - --header 'content-type: application/json' \ - --data-raw "{ - \"apiKey\": \"$API_KEY\", - \"dockerImageUri\": \"$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG\", - \"functionName\": \"$IMAGE_TAG\" - }" - - deploy-production: - name: Deploy Production - needs: deploy-staging - runs-on: ubuntu-latest - environment: production - env: - ECR_REPOSITORY: lambda-feedback-production-functions-repository - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set config.json output - id: set_config_var - run: | - content=`cat ./config.json` - # the following lines are only required for multi line json - content="${content//'%'/'%25'}" - content="${content//$'\n'/'%0A'}" - content="${content//$'\r'/'%0D'}" - # end of optional handling for multi line json - echo "::set-output name=configJson::$content" - - - name: set Evaluation Function Name - id: set_function_name - run: | - functionName="${{fromJson(steps.set_config_var.outputs.configJson).EvaluationFunctionName}}" - [[ -z "$functionName" ]] && { echo "Add EvaluationFunctionName to config.json" ; exit 1; } - echo "::set-output name=function_name::$functionName" - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} - aws-secret-access-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET }} - aws-region: eu-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - # Build docker image from algorithm, schema and requirements - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG app/ - # Push image to ECR - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" - - - name: deploy evaluation function - id: deploy-evaluation-function - env: - BACKEND_API_URL: https://prod-api.lambdafeedback.com - API_KEY: ${{ secrets.FUNCTION_ADMIN_API_KEY }} - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - curl --location --request POST "$BACKEND_API_URL/grading-function/ensure" \ - --header 'content-type: application/json' \ - --data-raw "{ - \"apiKey\": \"$API_KEY\", - \"dockerImageUri\": \"$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG\", - \"functionName\": \"$IMAGE_TAG\" - }" diff --git a/.github/workflows/test-lint.yml b/.github/workflows/test-lint.yml new file mode 100644 index 0000000..b9d4bce --- /dev/null +++ b/.github/workflows/test-lint.yml @@ -0,0 +1,55 @@ +name: Test and Lint + +on: + pull_request: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + checks: write + pull-requests: write + strategy: + fail-fast: false + matrix: + python-version: ["3.8"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m venv .venv + .venv/bin/pip install pytest + .venv/bin/pip install -r app/requirements.txt + + - name: Lint with flake8 + run: | + source .venv/bin/activate + .venv/bin/pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 ./app --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./app --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Run tests + if: always() + run: | + source .venv/bin/activate + pytest --junit-xml=./reports/pytest.xml --tb=auto -v + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results-${{ matrix.python-version }} + path: ./reports/pytest.xml + if-no-files-found: warn \ No newline at end of file diff --git a/README.md b/README.md index a8d3ebb..370fdc0 100755 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ This template repository contains the boilerplate code needed in order to create This version is specifically for python, however the ultimate goal is to make similar boilerplate repositories in any language, allowing tutors the freedom to code in what they feel most comfortable with. +## Deployment +[![Create Release Request](https://img.shields.io/badge/Create%20Release%20Request-blue?style=for-the-badge)](https://github.com/lambda-feedback/{REPO_NAME_HERE}/issues/new?template=release-request.yml) +To deploy to production, update the README button above to point to the correct repository. + ## Table of Contents - [Evaluation Function Template Repository](#evaluation-function-template-repository) - [Table of Contents](#table-of-contents) From d5b3a81b98d1c2d465e73eb14ff94325eed12554 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Wed, 6 May 2026 11:56:23 +0100 Subject: [PATCH 2/3] Disable database tests in the production deploy workflow. --- .github/workflows/production-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml index ec071c3..cdcc6c4 100644 --- a/.github/workflows/production-deploy.yml +++ b/.github/workflows/production-deploy.yml @@ -42,7 +42,7 @@ jobs: environment: "production" version-bump: ${{ inputs.version-bump }} branch: ${{ inputs.branch }} - run-database-tests: true + run-database-tests: false seed: ${{ inputs.seed }} request_delay: ${{ inputs.request_delay }} max_concurrency: ${{ inputs.max_concurrency }} From c4a316bedfa85936e8aa8d2a9cb624f383f08593 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Wed, 6 May 2026 12:17:15 +0100 Subject: [PATCH 3/3] Renamed test files for consistency and updated Python version to 3.12 in GitHub Actions workflow. --- .github/workflows/test-lint.yml | 2 +- README.md | 4 ++-- app/Dockerfile | 4 ++-- app/{evaluation_tests.py => evaluation_test.py} | 0 app/{preview_tests.py => preview_test.py} | 0 5 files changed, 5 insertions(+), 5 deletions(-) rename app/{evaluation_tests.py => evaluation_test.py} (100%) rename app/{preview_tests.py => preview_test.py} (100%) diff --git a/.github/workflows/test-lint.yml b/.github/workflows/test-lint.yml index b9d4bce..0f1e91d 100644 --- a/.github/workflows/test-lint.yml +++ b/.github/workflows/test-lint.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8"] + python-version: ["3.12"] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/README.md b/README.md index 370fdc0..42097c7 100755 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ app/ evaluation.py # Script containing the main evaluation_function preview.py # Script containing the preview_function docs.md # Documentation page for this function (required) - evaluation_tests.py # Unittests for the main evaluation_function - preview_tests.py # Unittests for the preview_function + evaluation_test.py # Unittests for the main evaluation_function + preview_test.py # Unittests for the preview_function requirements.txt # list of packages needed for algorithm.py Dockerfile # for building whole image to deploy to AWS diff --git a/app/Dockerfile b/app/Dockerfile index ff227d8..5882aff 100755 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -10,11 +10,11 @@ RUN pip3 install -r requirements.txt # Copy the evaluation and testing scripts COPY evaluation.py ./app/ -COPY evaluation_tests.py ./app/ +COPY evaluation_test.py ./app/ # Copy the preview and testing scripts COPY preview.py ./app/ -COPY preview_tests.py ./app/ +COPY preview_test.py ./app/ # Copy Documentation COPY docs/dev.md ./app/docs/dev.md diff --git a/app/evaluation_tests.py b/app/evaluation_test.py similarity index 100% rename from app/evaluation_tests.py rename to app/evaluation_test.py diff --git a/app/preview_tests.py b/app/preview_test.py similarity index 100% rename from app/preview_tests.py rename to app/preview_test.py