diff --git a/.github/ISSUE_TEMPLATE/release-request.yml b/.github/ISSUE_TEMPLATE/release-request.yml new file mode 100644 index 0000000..2ed9717 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/release-request.yml @@ -0,0 +1,53 @@ +name: "Release Request" +description: "Request a deployment by specifying the evaluation function, changes, target commit/branch, and test confirmation." +title: "Release Request" +labels: + - release + - deployment +assignees: [] +body: + - type: textarea + id: description_of_changes + attributes: + label: Description of changes + description: "Summarize what is changing and why. Include links to PRs, issues, or changelogs." + placeholder: | + - What changed: + - Why: + - Related PRs/Issues: #123, #456 + render: markdown + validations: + required: true + + - type: input + id: branch_to_deploy + attributes: + label: Branch to deploy + description: | + Specify Branch name to deploy. + placeholder: "e.g., release/2025-09-29" + validations: + required: true + + - type: dropdown + id: version-bump + attributes: + label: "๐Ÿš€ What kind of update is this?" + description: "Tell us how significant this change is. This helps us set the correct new version number." + options: + - "Patch: A small fix for a bug. It won't break anything for existing users. (e.g., 1.2.3 โž” 1.2.4)" + - "Minor: Adds a new feature, but doesn't change how existing ones work. A safe update. (e.g., 1.2.3 โž” 1.3.0)" + - "Major: A big change that alters existing features. Users may need to update their work to adapt. (e.g., 1.2.3 โž” 2.0.0)" + default: 0 + validations: + required: true + + - type: markdown + attributes: + value: | + --- + ### โšก Click the Link Below to Run the Workflow + + Clicking the link will take you to the Actions page. You will need to click the **"Run workflow"** button there to start the process. + + ## [โžก๏ธ Go to Workflow Run Page](../actions/workflows/production-deploy.yml) \ No newline at end of file diff --git a/.github/workflows/pre_production_tests.yml b/.github/workflows/pre_production_tests.yml new file mode 100644 index 0000000..2daabd1 --- /dev/null +++ b/.github/workflows/pre_production_tests.yml @@ -0,0 +1,49 @@ +name: Run Pre-Production Validation Tests + +on: + workflow_dispatch: + inputs: + eval_function: + type: string + description: "The name of the evaluation function to test" + required: true + sql_limit: + type: number + description: "The maximum number of SQL test cases to run" + required: false + default: 500 + seed: + type: string + description: "Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate." + required: false + default: '' + request_delay: + type: string + description: "Delay (seconds) between dispatching requests" + required: false + default: '0' + max_concurrency: + type: string + description: "Max concurrent requests (lower for GPT-backed functions)" + required: false + default: '5' + +jobs: + run-pre-production-tests: + name: ๐Ÿงช Run Staging Validation Tests + uses: lambda-feedback/Database-Testing/.github/workflows/test_evaluation_function.yml@main + with: + eval_function: ${{ inputs.eval_function }} + sql_limit: ${{ inputs.sql_limit }} + seed: ${{ inputs.seed }} + request_delay: ${{ inputs.request_delay }} + max_concurrency: ${{ inputs.max_concurrency }} + secrets: + TEST_API_ENDPOINT: ${{ secrets.TEST_API_ENDPOINT }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_HOST: ${{ secrets.DB_HOST }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_NAME: ${{ secrets.DB_NAME }} + GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_DB_CREDS }} + GCP_PROJECT_ID: ${{ secrets.GCP_DB_PROJECT_ID }} \ No newline at end of file diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml new file mode 100644 index 0000000..cdcc6c4 --- /dev/null +++ b/.github/workflows/production-deploy.yml @@ -0,0 +1,62 @@ +name: Deploy to Production + +on: + workflow_dispatch: + inputs: + version-bump: + description: 'Version bump type' + required: true + type: choice + options: + - patch + - minor + - major + default: patch + branch: + description: 'Branch to release from' + required: true + type: string + default: 'main' + seed: + description: 'Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate.' + required: false + type: string + default: '' + request_delay: + description: 'Delay (seconds) between dispatching requests' + required: false + type: string + default: '0' + max_concurrency: + description: 'Max concurrent requests (lower for GPT-backed functions)' + required: false + type: string + default: '5' +jobs: + deploy: + uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@deploy-request + with: + template-repository-name: 'lambda-feedback/evaluation-function-boilerplate-python' + build-file: "app/Dockerfile" + build-context: "./app" + environment: "production" + version-bump: ${{ inputs.version-bump }} + branch: ${{ inputs.branch }} + run-database-tests: false + seed: ${{ inputs.seed }} + request_delay: ${{ inputs.request_delay }} + max_concurrency: ${{ inputs.max_concurrency }} + + secrets: + aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} + aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} + function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY}} + gcp_credentials: ${{ secrets.GCP_DEPLOY }} + TEST_API_ENDPOINT: ${{ secrets.TEST_API_ENDPOINT }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_HOST: ${{ secrets.DB_HOST }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_NAME: ${{ secrets.DB_NAME }} + GCP_DB_CREDS: ${{ secrets.GCP_DB_CREDS }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} \ No newline at end of file diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml new file mode 100644 index 0000000..b03a9ec --- /dev/null +++ b/.github/workflows/staging-deploy.yml @@ -0,0 +1,74 @@ +name: Deploy to Staging + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + checks: write + pull-requests: write + strategy: + fail-fast: false + matrix: + python-version: ["3.8"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m venv .venv + .venv/bin/pip install pytest + .venv/bin/pip install -r app/requirements.txt + + + - name: Lint with flake8 + run: | + source .venv/bin/activate + .venv/bin/pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 ./app --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./app --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Run tests + if: always() + run: | + source .venv/bin/activate + pytest --junit-xml=./reports/pytest.xml --tb=auto -v + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results-${{ matrix.python-version }} + path: ./reports/pytest.xml + if-no-files-found: warn + deploy: + needs: test + uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@deploy-request + with: + template-repository-name: "lambda-feedback/evaluation-function-boilerplate-python" + build-file: "app/Dockerfile" + build-context: "./app" + build-platforms: "aws" + environment: "staging" + lfs: false + secrets: + aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} + aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} + function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY}} + gcp_credentials: ${{ secrets.GCP_DEPLOY }} diff --git a/.github/workflows/test-and-deploy.yml b/.github/workflows/test-and-deploy.yml deleted file mode 100755 index bbd6a9e..0000000 --- a/.github/workflows/test-and-deploy.yml +++ /dev/null @@ -1,193 +0,0 @@ -name: Test & Deploy Evaluation Function to AWS Lambda - -on: - push: - branches: - - master - - main - workflow_dispatch: - -jobs: - test: - name: Test - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: [3.8] - - defaults: - run: - working-directory: app/ - - env: - SCHEMAS_URL: https://raw.githubusercontent.com/lambda-feedback/request-response-schemas/master/ - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - python -m pip install -r requirements.txt - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Test Evaluation Function - run: | - pytest -v evaluation_tests.py::TestEvaluationFunction - - - name: Test Preview Function - run: | - pytest -v preview_tests.py::TestPreviewFunction - - deploy-staging: - name: Deploy Staging - needs: test - runs-on: ubuntu-latest - environment: production - env: - ECR_REPOSITORY: lambda-feedback-staging-functions-repository - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set config.json output - id: set_config_var - run: | - content=`cat ./config.json` - # the following lines are only required for multi line json - content="${content//'%'/'%25'}" - content="${content//$'\n'/'%0A'}" - content="${content//$'\r'/'%0D'}" - # end of optional handling for multi line json - echo "::set-output name=configJson::$content" - - - name: set Evaluation Function Name - id: set_function_name - run: | - functionName="${{fromJson(steps.set_config_var.outputs.configJson).EvaluationFunctionName}}" - [[ -z "$functionName" ]] && { echo "Add EvaluationFunctionName to config.json" ; exit 1; } - echo "::set-output name=function_name::$functionName" - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} - aws-secret-access-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET }} - aws-region: eu-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - # Build docker image from algorithm, schema and requirements - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG app/ - # Push image to ECR - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" - - - name: deploy evaluation function - id: deploy-evaluation-function - env: - BACKEND_API_URL: https://staging-api.lambdafeedback.com - API_KEY: ${{ secrets.FUNCTION_ADMIN_API_KEY }} - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - curl --location --request POST "$BACKEND_API_URL/grading-function/ensure" \ - --header 'content-type: application/json' \ - --data-raw "{ - \"apiKey\": \"$API_KEY\", - \"dockerImageUri\": \"$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG\", - \"functionName\": \"$IMAGE_TAG\" - }" - - deploy-production: - name: Deploy Production - needs: deploy-staging - runs-on: ubuntu-latest - environment: production - env: - ECR_REPOSITORY: lambda-feedback-production-functions-repository - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set config.json output - id: set_config_var - run: | - content=`cat ./config.json` - # the following lines are only required for multi line json - content="${content//'%'/'%25'}" - content="${content//$'\n'/'%0A'}" - content="${content//$'\r'/'%0D'}" - # end of optional handling for multi line json - echo "::set-output name=configJson::$content" - - - name: set Evaluation Function Name - id: set_function_name - run: | - functionName="${{fromJson(steps.set_config_var.outputs.configJson).EvaluationFunctionName}}" - [[ -z "$functionName" ]] && { echo "Add EvaluationFunctionName to config.json" ; exit 1; } - echo "::set-output name=function_name::$functionName" - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} - aws-secret-access-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET }} - aws-region: eu-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - # Build docker image from algorithm, schema and requirements - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG app/ - # Push image to ECR - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" - - - name: deploy evaluation function - id: deploy-evaluation-function - env: - BACKEND_API_URL: https://prod-api.lambdafeedback.com - API_KEY: ${{ secrets.FUNCTION_ADMIN_API_KEY }} - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.set_function_name.outputs.function_name }} - run: | - curl --location --request POST "$BACKEND_API_URL/grading-function/ensure" \ - --header 'content-type: application/json' \ - --data-raw "{ - \"apiKey\": \"$API_KEY\", - \"dockerImageUri\": \"$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG\", - \"functionName\": \"$IMAGE_TAG\" - }" diff --git a/.github/workflows/test-lint.yml b/.github/workflows/test-lint.yml new file mode 100644 index 0000000..0f1e91d --- /dev/null +++ b/.github/workflows/test-lint.yml @@ -0,0 +1,55 @@ +name: Test and Lint + +on: + pull_request: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + checks: write + pull-requests: write + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m venv .venv + .venv/bin/pip install pytest + .venv/bin/pip install -r app/requirements.txt + + - name: Lint with flake8 + run: | + source .venv/bin/activate + .venv/bin/pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 ./app --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./app --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Run tests + if: always() + run: | + source .venv/bin/activate + pytest --junit-xml=./reports/pytest.xml --tb=auto -v + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results-${{ matrix.python-version }} + path: ./reports/pytest.xml + if-no-files-found: warn \ No newline at end of file diff --git a/README.md b/README.md index a8d3ebb..42097c7 100755 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ This template repository contains the boilerplate code needed in order to create This version is specifically for python, however the ultimate goal is to make similar boilerplate repositories in any language, allowing tutors the freedom to code in what they feel most comfortable with. +## Deployment +[![Create Release Request](https://img.shields.io/badge/Create%20Release%20Request-blue?style=for-the-badge)](https://github.com/lambda-feedback/{REPO_NAME_HERE}/issues/new?template=release-request.yml) +To deploy to production, update the README button above to point to the correct repository. + ## Table of Contents - [Evaluation Function Template Repository](#evaluation-function-template-repository) - [Table of Contents](#table-of-contents) @@ -25,8 +29,8 @@ app/ evaluation.py # Script containing the main evaluation_function preview.py # Script containing the preview_function docs.md # Documentation page for this function (required) - evaluation_tests.py # Unittests for the main evaluation_function - preview_tests.py # Unittests for the preview_function + evaluation_test.py # Unittests for the main evaluation_function + preview_test.py # Unittests for the preview_function requirements.txt # list of packages needed for algorithm.py Dockerfile # for building whole image to deploy to AWS diff --git a/app/Dockerfile b/app/Dockerfile index ff227d8..5882aff 100755 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -10,11 +10,11 @@ RUN pip3 install -r requirements.txt # Copy the evaluation and testing scripts COPY evaluation.py ./app/ -COPY evaluation_tests.py ./app/ +COPY evaluation_test.py ./app/ # Copy the preview and testing scripts COPY preview.py ./app/ -COPY preview_tests.py ./app/ +COPY preview_test.py ./app/ # Copy Documentation COPY docs/dev.md ./app/docs/dev.md diff --git a/app/evaluation_tests.py b/app/evaluation_test.py similarity index 100% rename from app/evaluation_tests.py rename to app/evaluation_test.py diff --git a/app/preview_tests.py b/app/preview_test.py similarity index 100% rename from app/preview_tests.py rename to app/preview_test.py