Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .flake8
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[flake8]
extend-ignore = C901, W504, E203
extend-ignore = C901, W504, E203, E701
max-line-length = 130
# NOTE: Update in .pre-commit-config.yaml as well
extend-exclude = .git,__pycache__,old,build,dist,*/migrations/*.py,.venv
Expand Down
211 changes: 25 additions & 186 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,191 +1,30 @@
name: Python check
name: CI

on:
workflow_call:
inputs:
push_docker_image:
type: string # true or false
default: "false"
outputs:
docker_image_name:
description: "Only docker image name"
value: ${{ jobs.build_test.outputs.docker_image_name }}
docker_image_tag:
description: "Only docker image tag"
value: ${{ jobs.build_test.outputs.docker_image_tag }}
docker_image:
description: "docker image with tag"
value: ${{ jobs.build_test.outputs.docker_image }}
pull_request:
# NOTE: For other, they should be run through helm github action ./helm-publish.yml

jobs:
pre_commit_checks:
name: Pre-Commit checks
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@main

- uses: actions/setup-python@v5
with:
python-version-file: '.python-version'

- uses: astral-sh/setup-uv@v5
with:
enable-cache: true

- uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: gdal-bin libgdal-dev

- name: Setup uv python environment
run: uv venv

- name: uv lock check
run: uv lock --locked --offline

- name: uv sync
run: uv sync --all-extras

- uses: pre-commit/action@main

build_test:
name: Test
runs-on: ubuntu-latest
needs: pre_commit_checks

outputs:
docker_image_name: ${{ steps.prep.outputs.tagged_image_name }}
docker_image_tag: ${{ steps.prep.outputs.tag }}
docker_image: ${{ steps.prep.outputs.tagged_image }}

steps:
- uses: actions/checkout@main

- name: Login to GitHub Container Registry
uses: docker/login-action@v3
if: ${{ inputs.push_docker_image }}
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: 🐳 Prepare Docker
id: prep
env:
IMAGE_NAME: ghcr.io/${{ github.repository }}
run: |
BRANCH_NAME=$(echo $GITHUB_REF_NAME | sed 's|:|-|' | tr '[:upper:]' '[:lower:]' | sed 's/_/-/g' | cut -c1-100 | sed 's/-*$//')

# XXX: Check if there is a slash in the BRANCH_NAME eg: project/add-docker
if [[ "$BRANCH_NAME" == *"/"* ]]; then
# XXX: Change the docker image package to -alpha
IMAGE_NAME="$IMAGE_NAME-alpha"
TAG="$(echo "$BRANCH_NAME" | sed 's|/|-|g').c$(echo $GITHUB_SHA | head -c7)"
else
TAG="$BRANCH_NAME.c$(echo $GITHUB_SHA | head -c7)"
fi

IMAGE_NAME=$(echo $IMAGE_NAME | tr '[:upper:]' '[:lower:]')
echo "tagged_image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT
echo "tag=${TAG}" >> $GITHUB_OUTPUT
echo "tagged_image=${IMAGE_NAME}:${TAG}" >> $GITHUB_OUTPUT
echo "::notice::Tagged docker image: ${IMAGE_NAME}:${TAG}"

- name: 🐳 Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3

- name: 🐳 Cache Docker layers
uses: actions/cache@v4
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.ref }}
restore-keys: |
${{ runner.os }}-buildx-refs/develop
${{ runner.os }}-buildx-

- name: 🐳 Docker build
uses: docker/build-push-action@v6
with:
context: .
builder: ${{ steps.buildx.outputs.name }}
file: Dockerfile
load: true
push: false
tags: ${{ steps.prep.outputs.tagged_image }}
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new

- name: 🕮 Validate if there are no pending django migrations.
env:
DOCKER_IMAGE_SERVER: ${{ steps.prep.outputs.tagged_image }}
run: |
docker compose -f ./gh-docker-compose.yml run --rm server bash -c 'wait-for-it db:5432 && ./manage.py makemigrations --check --dry-run' || {
echo 'There are some changes to be reflected in the migration. Make sure to run makemigrations';
exit 1;
}

- name: 🕮 Validate SentryMonitor config
env:
DOCKER_IMAGE_SERVER: ${{ steps.prep.outputs.tagged_image }}
run: |
docker compose -f ./gh-docker-compose.yml run --rm server ./manage.py setup_sentry_cron_monitor --validate-only || {
echo 'There are some changes to be reflected in the SentryMonitor. Make sure to update SentryMonitor';
exit 1;
}

- name: Run django migrations
env:
DOCKER_IMAGE_SERVER: ${{ steps.prep.outputs.tagged_image }}
run: docker compose -f ./gh-docker-compose.yml run --rm server ./manage.py test --keepdb -v 2 --pattern="test_fake.py"

# NOTE: Schema generation requires a valid database. Therefore, this step must run after "Run Django migrations."
- name: 🕮 Validate latest openapi schema
env:
DOCKER_IMAGE_SERVER: ${{ steps.prep.outputs.tagged_image }}
run: |
docker compose -f ./gh-docker-compose.yml run --rm server ./manage.py spectacular --file /ci-share/openapi-schema-latest.yaml &&
cmp --silent openapi-schema.yaml ./ci-share/openapi-schema-latest.yaml || {
echo 'The openapi-schema.yaml is not up to date with the latest changes. Please update and push latest';
diff openapi-schema.yaml ./ci-share/openapi-schema-latest.yaml;
exit 1;
}

- name: 🐳 Docker push
if: ${{ inputs.push_docker_image }}
uses: docker/build-push-action@v6
with:
tags: ${{ steps.prep.outputs.tagged_image }}
push: true

# Temp fix
# https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#github-cache
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- name: 🐳 Move docker cache (🧙 Hack fix)
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache

validate_helm:
name: Validate Helm
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@main

- name: Install Helm
uses: azure/setup-helm@v4

- name: 🐳 Helm dependency
run: |
yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock | sh --
helm dependency build ./helm

- name: 🐳 Helm lint
run: helm lint ./helm --values ./helm/values-test.yaml

- name: 🐳 Helm template
run: helm template ./helm --values ./helm/values-test.yaml
jobs:
_:
uses: toggle-corp/toggle-django-action/.github/workflows/pipeline.yml@v0.1.0-dev0
with:
# Job: Pre-commit
pre_commit__apt_packages: "gdal-bin libgdal-dev"
# Job: Docker
docker__wait_cmd: "./manage.py wait_for_resources --db --redis"
docker__migration_dummy_test: "risk_module.tests.FakeTest"
docker__run_graphql_check: false
docker__run_openapi_check: true
docker__run_openapi_check_schema_filepath: "openapi-schema.yaml"
docker__compose_file: "gh-docker-compose.yml"
docker__compose_service_name: "server"
docker__compose_test_cmd: "ls" # NOTE: This is no tests
# -- Extra steps
docker__post_test__extra_step_01: "🕮 Validate SentryMonitor config"
docker__post_test__extra_step_01_command: |
docker compose run --rm server ./manage.py setup_sentry_cron_monitor --validate-only || {
echo 'There are some changes to be reflected in the SentryMonitor. Make sure to update SentryMonitor';
exit 1;
}
# Job: Helm
helm__chart_directory: "./helm"
11 changes: 11 additions & 0 deletions .github/workflows/git.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
name: Lint commits

on: [pull_request]

jobs:
lint:
name: Commit Lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@main
- uses: toggle-corp/commit-lint@main
99 changes: 27 additions & 72 deletions .github/workflows/helm-publish.yml
Original file line number Diff line number Diff line change
@@ -1,82 +1,37 @@
name: Helm publish
name: Publish

on:
workflow_dispatch:
push:
branches:
- develop
- project/*
# XXX: To add tags: Update the -alpha logic

permissions:
packages: write

jobs:
ci:
name: CI
uses: ./.github/workflows/ci.yml
_:
uses: toggle-corp/toggle-django-action/.github/workflows/pipeline.yml@v0.1.0-dev0
with:
push_docker_image: true

build:
name: Publish Helm
needs: ci
runs-on: ubuntu-latest

steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Install Helm
uses: azure/setup-helm@v3

- name: 🐳 Helm dependency
run: |
yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock | sh --
helm dependency build ./helm

- name: Tag docker image in Helm Chart values.yaml
env:
IMAGE_NAME: ${{ needs.ci.outputs.docker_image_name }}
IMAGE_TAG: ${{ needs.ci.outputs.docker_image_tag }}
run: |
# Update values.yaml with latest docker image
sed -i "s|SET-BY-CICD-IMAGE|$IMAGE_NAME|" helm/values.yaml
sed -i "s/SET-BY-CICD-TAG/$IMAGE_TAG/" helm/values.yaml

- name: Package Helm Chart
id: set-variables
run: |
# XXX: Check if there is a slash in the BRANCH_NAME eg: project/add-docker
if [[ "$GITHUB_REF_NAME" == *"/"* ]]; then
# XXX: Change the helm chart to <chart-name>-alpha
sed -i 's/^name: \(.*\)/name: \1-alpha/' helm/Chart.yaml
fi

SHA_SHORT=$(git rev-parse --short HEAD)
sed -i "s/SET-BY-CICD/$SHA_SHORT/g" helm/Chart.yaml
helm package ./helm -d .helm-charts

- name: Push Helm Chart
env:
IMAGE: ${{ needs.ci.outputs.docker_image }}
OCI_REPO: oci://ghcr.io/${{ github.repository }}
run: |
OCI_REPO=$(echo $OCI_REPO | tr '[:upper:]' '[:lower:]')
PACKAGE_FILE=$(ls .helm-charts/*.tgz | head -n 1)
echo "# Helm Chart" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Tagged Image: **$IMAGE**" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Helm push output" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```bash' >> $GITHUB_STEP_SUMMARY
helm push "$PACKAGE_FILE" $OCI_REPO >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
# -- Same as validate
# Job: Pre-commit
pre_commit__apt_packages: "gdal-bin libgdal-dev"
# Job: Docker
docker__wait_cmd: "./manage.py wait_for_resources --db --redis"
docker__migration_dummy_test: "risk_module.tests.FakeTest"
docker__run_graphql_check: false
docker__run_openapi_check: true
docker__run_openapi_check_schema_filepath: "openapi-schema.yaml"
docker__compose_file: "gh-docker-compose.yml"
docker__compose_service_name: "server"
docker__compose_test_cmd: "ls" # NOTE: This is no tests
# -- Extra steps
docker__post_test__extra_step_01: "🕮 Validate SentryMonitor config"
docker__post_test__extra_step_01_command: |
docker compose run --rm server ./manage.py setup_sentry_cron_monitor --validate-only || {
echo 'There are some changes to be reflected in the SentryMonitor. Make sure to update SentryMonitor';
exit 1;
}
# Job: Helm
helm__chart_directory: "./helm"
# -- Publish
docker__push: true
helm__push: true
4 changes: 4 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
[submodule "fugit"]
path = fugit
url = git@github.com:toggle-corp/fugit.git
branch = v0.1.1
Loading
Loading