diff --git a/.flake8 b/.flake8 index 7fc3dd5..9776cf2 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -extend-ignore = C901, W504, E203 +extend-ignore = C901, W504, E203, E701 max-line-length = 130 # NOTE: Update in .pre-commit-config.yaml as well extend-exclude = .git,__pycache__,old,build,dist,*/migrations/*.py,.venv diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e02f299..c2f199c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,191 +1,30 @@ -name: Python check +name: CI on: - workflow_call: - inputs: - push_docker_image: - type: string # true or false - default: "false" - outputs: - docker_image_name: - description: "Only docker image name" - value: ${{ jobs.build_test.outputs.docker_image_name }} - docker_image_tag: - description: "Only docker image tag" - value: ${{ jobs.build_test.outputs.docker_image_tag }} - docker_image: - description: "docker image with tag" - value: ${{ jobs.build_test.outputs.docker_image }} pull_request: - # NOTE: For other, they should be run through helm github action ./helm-publish.yml -jobs: - pre_commit_checks: - name: Pre-Commit checks - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@main - - - uses: actions/setup-python@v5 - with: - python-version-file: '.python-version' - - - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: gdal-bin libgdal-dev - - - name: Setup uv python environment - run: uv venv - - - name: uv lock check - run: uv lock --locked --offline - - - name: uv sync - run: uv sync --all-extras - - - uses: pre-commit/action@main - - build_test: - name: Test - runs-on: ubuntu-latest - needs: pre_commit_checks - - outputs: - docker_image_name: ${{ steps.prep.outputs.tagged_image_name }} - docker_image_tag: ${{ steps.prep.outputs.tag }} - docker_image: ${{ steps.prep.outputs.tagged_image }} - - steps: - - uses: actions/checkout@main - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - if: ${{ inputs.push_docker_image }} - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: 🐳 Prepare Docker - id: prep - env: - IMAGE_NAME: ghcr.io/${{ github.repository }} - run: | - BRANCH_NAME=$(echo $GITHUB_REF_NAME | sed 's|:|-|' | tr '[:upper:]' '[:lower:]' | sed 's/_/-/g' | cut -c1-100 | sed 's/-*$//') - - # XXX: Check if there is a slash in the BRANCH_NAME eg: project/add-docker - if [[ "$BRANCH_NAME" == *"/"* ]]; then - # XXX: Change the docker image package to -alpha - IMAGE_NAME="$IMAGE_NAME-alpha" - TAG="$(echo "$BRANCH_NAME" | sed 's|/|-|g').c$(echo $GITHUB_SHA | head -c7)" - else - TAG="$BRANCH_NAME.c$(echo $GITHUB_SHA | head -c7)" - fi - - IMAGE_NAME=$(echo $IMAGE_NAME | tr '[:upper:]' '[:lower:]') - echo "tagged_image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT - echo "tag=${TAG}" >> $GITHUB_OUTPUT - echo "tagged_image=${IMAGE_NAME}:${TAG}" >> $GITHUB_OUTPUT - echo "::notice::Tagged docker image: ${IMAGE_NAME}:${TAG}" - - name: 🐳 Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: 🐳 Cache Docker layers - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.ref }} - restore-keys: | - ${{ runner.os }}-buildx-refs/develop - ${{ runner.os }}-buildx- - - - name: 🐳 Docker build - uses: docker/build-push-action@v6 - with: - context: . - builder: ${{ steps.buildx.outputs.name }} - file: Dockerfile - load: true - push: false - tags: ${{ steps.prep.outputs.tagged_image }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - - name: 🕮 Validate if there are no pending django migrations. - env: - DOCKER_IMAGE_SERVER: ${{ steps.prep.outputs.tagged_image }} - run: | - docker compose -f ./gh-docker-compose.yml run --rm server bash -c 'wait-for-it db:5432 && ./manage.py makemigrations --check --dry-run' || { - echo 'There are some changes to be reflected in the migration. Make sure to run makemigrations'; - exit 1; - } - - - name: 🕮 Validate SentryMonitor config - env: - DOCKER_IMAGE_SERVER: ${{ steps.prep.outputs.tagged_image }} - run: | - docker compose -f ./gh-docker-compose.yml run --rm server ./manage.py setup_sentry_cron_monitor --validate-only || { - echo 'There are some changes to be reflected in the SentryMonitor. Make sure to update SentryMonitor'; - exit 1; - } - - - name: Run django migrations - env: - DOCKER_IMAGE_SERVER: ${{ steps.prep.outputs.tagged_image }} - run: docker compose -f ./gh-docker-compose.yml run --rm server ./manage.py test --keepdb -v 2 --pattern="test_fake.py" - - # NOTE: Schema generation requires a valid database. Therefore, this step must run after "Run Django migrations." - - name: 🕮 Validate latest openapi schema - env: - DOCKER_IMAGE_SERVER: ${{ steps.prep.outputs.tagged_image }} - run: | - docker compose -f ./gh-docker-compose.yml run --rm server ./manage.py spectacular --file /ci-share/openapi-schema-latest.yaml && - cmp --silent openapi-schema.yaml ./ci-share/openapi-schema-latest.yaml || { - echo 'The openapi-schema.yaml is not up to date with the latest changes. Please update and push latest'; - diff openapi-schema.yaml ./ci-share/openapi-schema-latest.yaml; - exit 1; - } - - - name: 🐳 Docker push - if: ${{ inputs.push_docker_image }} - uses: docker/build-push-action@v6 - with: - tags: ${{ steps.prep.outputs.tagged_image }} - push: true - - # Temp fix - # https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#github-cache - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: 🐳 Move docker cache (🧙 Hack fix) - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - validate_helm: - name: Validate Helm - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@main - - - name: Install Helm - uses: azure/setup-helm@v4 - - - name: 🐳 Helm dependency - run: | - yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock | sh -- - helm dependency build ./helm - - - name: 🐳 Helm lint - run: helm lint ./helm --values ./helm/values-test.yaml - - - name: 🐳 Helm template - run: helm template ./helm --values ./helm/values-test.yaml +jobs: + _: + uses: toggle-corp/toggle-django-action/.github/workflows/pipeline.yml@v0.1.0-dev0 + with: + # Job: Pre-commit + pre_commit__apt_packages: "gdal-bin libgdal-dev" + # Job: Docker + docker__wait_cmd: "./manage.py wait_for_resources --db --redis" + docker__migration_dummy_test: "risk_module.tests.FakeTest" + docker__run_graphql_check: false + docker__run_openapi_check: true + docker__run_openapi_check_schema_filepath: "openapi-schema.yaml" + docker__compose_file: "gh-docker-compose.yml" + docker__compose_service_name: "server" + docker__compose_test_cmd: "ls" # NOTE: This is no tests + # -- Extra steps + docker__post_test__extra_step_01: "🕮 Validate SentryMonitor config" + docker__post_test__extra_step_01_command: | + docker compose run --rm server ./manage.py setup_sentry_cron_monitor --validate-only || { + echo 'There are some changes to be reflected in the SentryMonitor. Make sure to update SentryMonitor'; + exit 1; + } + # Job: Helm + helm__chart_directory: "./helm" diff --git a/.github/workflows/git.yml b/.github/workflows/git.yml new file mode 100644 index 0000000..52c6062 --- /dev/null +++ b/.github/workflows/git.yml @@ -0,0 +1,11 @@ +name: Lint commits + +on: [pull_request] + +jobs: + lint: + name: Commit Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@main + - uses: toggle-corp/commit-lint@main diff --git a/.github/workflows/helm-publish.yml b/.github/workflows/helm-publish.yml index 42163eb..56cd51e 100644 --- a/.github/workflows/helm-publish.yml +++ b/.github/workflows/helm-publish.yml @@ -1,4 +1,4 @@ -name: Helm publish +name: Publish on: workflow_dispatch: @@ -6,77 +6,32 @@ on: branches: - develop - project/* - # XXX: To add tags: Update the -alpha logic - -permissions: - packages: write jobs: - ci: - name: CI - uses: ./.github/workflows/ci.yml + _: + uses: toggle-corp/toggle-django-action/.github/workflows/pipeline.yml@v0.1.0-dev0 with: - push_docker_image: true - - build: - name: Publish Helm - needs: ci - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Install Helm - uses: azure/setup-helm@v3 - - - name: 🐳 Helm dependency - run: | - yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock | sh -- - helm dependency build ./helm - - - name: Tag docker image in Helm Chart values.yaml - env: - IMAGE_NAME: ${{ needs.ci.outputs.docker_image_name }} - IMAGE_TAG: ${{ needs.ci.outputs.docker_image_tag }} - run: | - # Update values.yaml with latest docker image - sed -i "s|SET-BY-CICD-IMAGE|$IMAGE_NAME|" helm/values.yaml - sed -i "s/SET-BY-CICD-TAG/$IMAGE_TAG/" helm/values.yaml - - - name: Package Helm Chart - id: set-variables - run: | - # XXX: Check if there is a slash in the BRANCH_NAME eg: project/add-docker - if [[ "$GITHUB_REF_NAME" == *"/"* ]]; then - # XXX: Change the helm chart to -alpha - sed -i 's/^name: \(.*\)/name: \1-alpha/' helm/Chart.yaml - fi - - SHA_SHORT=$(git rev-parse --short HEAD) - sed -i "s/SET-BY-CICD/$SHA_SHORT/g" helm/Chart.yaml - helm package ./helm -d .helm-charts - - - name: Push Helm Chart - env: - IMAGE: ${{ needs.ci.outputs.docker_image }} - OCI_REPO: oci://ghcr.io/${{ github.repository }} - run: | - OCI_REPO=$(echo $OCI_REPO | tr '[:upper:]' '[:lower:]') - PACKAGE_FILE=$(ls .helm-charts/*.tgz | head -n 1) - echo "# Helm Chart" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Tagged Image: **$IMAGE**" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Helm push output" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```bash' >> $GITHUB_STEP_SUMMARY - helm push "$PACKAGE_FILE" $OCI_REPO >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY + # -- Same as validate + # Job: Pre-commit + pre_commit__apt_packages: "gdal-bin libgdal-dev" + # Job: Docker + docker__wait_cmd: "./manage.py wait_for_resources --db --redis" + docker__migration_dummy_test: "risk_module.tests.FakeTest" + docker__run_graphql_check: false + docker__run_openapi_check: true + docker__run_openapi_check_schema_filepath: "openapi-schema.yaml" + docker__compose_file: "gh-docker-compose.yml" + docker__compose_service_name: "server" + docker__compose_test_cmd: "ls" # NOTE: This is no tests + # -- Extra steps + docker__post_test__extra_step_01: "🕮 Validate SentryMonitor config" + docker__post_test__extra_step_01_command: | + docker compose run --rm server ./manage.py setup_sentry_cron_monitor --validate-only || { + echo 'There are some changes to be reflected in the SentryMonitor. Make sure to update SentryMonitor'; + exit 1; + } + # Job: Helm + helm__chart_directory: "./helm" + # -- Publish + docker__push: true + helm__push: true diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..49d5a2c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "fugit"] + path = fugit + url = git@github.com:toggle-corp/fugit.git + branch = v0.1.1 diff --git a/common/management/commands/wait_for_resources.py b/common/management/commands/wait_for_resources.py new file mode 100644 index 0000000..e8eb270 --- /dev/null +++ b/common/management/commands/wait_for_resources.py @@ -0,0 +1,137 @@ +import signal +import time +from urllib.parse import urljoin + +import requests +from django.conf import settings +from django.core.cache import cache +from django.core.management.base import BaseCommand +from django.db import connections +from django.db.utils import OperationalError +from redis.exceptions import ConnectionError as RedisConnectionError + + +class TimeoutException(Exception): ... + + +class RetryHelper: + def __init__(self, base_wait_seconds: int = 2, wait_max_seconds: int = 60): + self.base_wait_seconds = base_wait_seconds + self.wait_max_seconds = wait_max_seconds + self.attempt = 1 + self.next_wait = base_wait_seconds + self.start_time = time.time() + + def next_wait_seconds(self) -> int: + return self.next_wait + + def wait(self) -> None: + time.sleep(self.next_wait) + self.attempt += 1 + if self.next_wait < self.wait_max_seconds: + self.next_wait = self.base_wait_seconds**self.attempt + else: + self.next_wait = self.wait_max_seconds + + def total_time(self) -> float: + return time.time() - self.start_time + + def try_again_message(self, prefix: str) -> str: + return f"{prefix}, Attempt: {self.attempt}, try again after {self.next_wait_seconds()} seconds..." + + +def timeout_handler(*_): + raise Exception("The command timed out.") + + +class Command(BaseCommand): + help = "Wait for resources our application depends on" + + def wait_for_db(self): + self.stdout.write("Waiting for DB...") + db_conn = None + retry_helper = RetryHelper() + while True: + try: + db_conn = connections["default"] + db_conn.ensure_connection() + break + except OperationalError: + ... + # Try again + self.stdout.write(self.style.WARNING(retry_helper.try_again_message("DB not available"))) + retry_helper.wait() + + self.stdout.write(self.style.SUCCESS(f"DB is available after {retry_helper.total_time()} seconds")) + + def wait_for_redis(self): + self.stdout.write("Waiting for Redis...") + redis_conn = None + retry_helper = RetryHelper() + while True: + try: + cache.set("wait-for-it-ping", "pong", timeout=1) # Set a key to check Redis availability + redis_conn = cache.get("wait-for-it-ping") # Try to get the value back from Redis + if redis_conn != "pong": + raise TypeError + break + except (RedisConnectionError, TypeError): + ... + # Try again + self.stdout.write(self.style.WARNING(retry_helper.try_again_message("Redis not available"))) + retry_helper.wait() + + self.stdout.write(self.style.SUCCESS(f"Redis is available after {retry_helper.total_time()} seconds")) + + def wait_for_minio(self): + self.stdout.write("Waiting for Minio...") + endpoint_url = getattr(settings, "AWS_S3_PROXIES", {}).get("http") or getattr(settings, "AWS_S3_ENDPOINT_URL", None) + if endpoint_url is None: + self.stdout.write(self.style.WARNING("No endpoint_url is provided. Skipping wait")) + return + + retry_helper = RetryHelper() + while True: + try: + response = requests.get(urljoin(endpoint_url, "/minio/health/live"), timeout=5) + if response.status_code == 200: + break + except requests.exceptions.RequestException: + ... + # Try again + self.stdout.write(self.style.WARNING(retry_helper.try_again_message("Minio not available"))) + retry_helper.wait() + + self.stdout.write(self.style.SUCCESS(f"Minio is available after {retry_helper.total_time()} seconds")) + + def add_arguments(self, parser): + parser.add_argument( + "--timeout", + type=int, + default=600, + help="The maximum time (in seconds) the command is allowed to run before timing out. Default is 10 min.", + ) + parser.add_argument("--db", action="store_true", help="Wait for DB to be available") + parser.add_argument("--redis", action="store_true", help="Wait for Redis to be available") + parser.add_argument("--minio", action="store_true", help="Wait for MinIO (S3) storage to be available") + parser.add_argument("--all", action="store_true", help="Wait for all to be available") + + def handle(self, **kwargs): + timeout = kwargs["timeout"] + _all = kwargs["all"] + + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(timeout) + + try: + if _all or kwargs["db"]: + self.wait_for_db() + if _all or kwargs["minio"]: + self.wait_for_minio() + if _all or kwargs["redis"]: + self.wait_for_redis() + except TimeoutException: + ... + finally: + # Disable the alarm (cleanup) + signal.alarm(0) diff --git a/fugit b/fugit new file mode 160000 index 0000000..1460a27 --- /dev/null +++ b/fugit @@ -0,0 +1 @@ +Subproject commit 1460a27ff4048425fb3efb34dfc35ea1293fe8b7 diff --git a/gh-docker-compose.yml b/gh-docker-compose.yml index a5b869e..000bb87 100644 --- a/gh-docker-compose.yml +++ b/gh-docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.2' - services: db: image: postgres:15.0-alpine @@ -22,7 +20,7 @@ services: retries: 5 server: - image: $DOCKER_IMAGE_SERVER + image: $DOCKER_IMAGE_BACKEND environment: CI: "true" # https://github.com/pytest-dev/pytest/issues/7443 diff --git a/helm/.gitignore b/helm/.gitignore index 305c05f..295e8a9 100644 --- a/helm/.gitignore +++ b/helm/.gitignore @@ -1,2 +1,3 @@ -charts/ +charts values-local.yaml +.helm-charts/ diff --git a/helm/.helmignore b/helm/.helmignore new file mode 100644 index 0000000..2b7e4bd --- /dev/null +++ b/helm/.helmignore @@ -0,0 +1,5 @@ +values-local.yaml +.helm-charts +tests.yaml +tests/ +snapshots/ diff --git a/helm/Chart.lock b/helm/Chart.lock index 2e7d732..5b4e308 100644 --- a/helm/Chart.lock +++ b/helm/Chart.lock @@ -1,9 +1,6 @@ dependencies: -- name: postgresql - repository: https://charts.bitnami.com/bitnami - version: 15.5.32 -- name: redis - repository: https://charts.bitnami.com/bitnami - version: 20.0.4 -digest: sha256:059705d9a5ad3755db9ef9307145e85146664191bbaf86618ec8b13ec80ddaee -generated: "2024-11-20T12:21:02.641728835+05:45" +- name: toggle-django-helm + repository: oci://ghcr.io/toggle-corp + version: 0.2.5 +digest: sha256:94f65db722244c4ba17db79c144bc75bd43f4e21c4e221a3697a78c523fc4298 +generated: "2026-03-21T22:29:52.126868712+05:45" diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 71660f7..ea9daa3 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -3,17 +3,11 @@ name: ifrcgo-risk-module-helm description: "Helm Chart to deploy the IFRC GO Risk Module Infrastructure" type: application version: 0.0.1-SET-BY-CICD -# appVersion: "1.16.0" sources: - https://github.com/IFRCGo/go-risk-module-api - dependencies: - - name: postgresql - version: 15.5.32 - condition: postgresql.enabled - repository: https://charts.bitnami.com/bitnami - - name: redis - version: 20.0.4 - condition: redis.enabled - repository: https://charts.bitnami.com/bitnami + - name: toggle-django-helm + alias: app + version: 0.2.5 + repository: oci://ghcr.io/toggle-corp diff --git a/helm/snapshots/alpha-1.yaml b/helm/snapshots/alpha-1.yaml new file mode 100644 index 0000000..3f6f14b --- /dev/null +++ b/helm/snapshots/alpha-1.yaml @@ -0,0 +1,1405 @@ +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/networkpolicy.yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: fircgo-risk-module-minio + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: minio + policyTypes: + - Ingress + - Egress + egress: + - {} + ingress: + # Allow inbound connections + - ports: + - port: 9001 + - port: 9000 +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/provisioning-networkpolicy.yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: fircgo-risk-module-minio-provisioning + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 +spec: + podSelector: + matchLabels: + app.kubernetes.io/component: minio-provisioning + policyTypes: + - Ingress + - Egress + egress: + - {} + ingress: +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/postgresql/templates/primary/networkpolicy.yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: ifrcgo-risk-module-pg + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgresql + app.kubernetes.io/version: 17.2.0 + helm.sh/chart: postgresql-16.4.5 + app.kubernetes.io/component: primary +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql + app.kubernetes.io/component: primary + policyTypes: + - Ingress + - Egress + egress: + - {} + ingress: + - ports: + - port: 5432 +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/pdb.yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: fircgo-risk-module-minio + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: minio +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/postgresql/templates/primary/pdb.yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: ifrcgo-risk-module-pg + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgresql + app.kubernetes.io/version: 17.2.0 + helm.sh/chart: postgresql-16.4.5 + app.kubernetes.io/component: primary +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql + app.kubernetes.io/component: primary +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fircgo-risk-module-minio + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 +automountServiceAccountToken: false +secrets: + - name: fircgo-risk-module-minio +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/postgresql/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ifrcgo-risk-module-pg + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgresql + app.kubernetes.io/version: 17.2.0 + helm.sh/chart: postgresql-16.4.5 +automountServiceAccountToken: false +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: fircgo-risk-module-minio + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 +type: Opaque +data: + root-user: "cmlzay1tb2R1bGU=" + root-password: "WFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFg=" +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/postgresql/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: ifrcgo-risk-module-pg + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgresql + app.kubernetes.io/version: 17.2.0 + helm.sh/chart: postgresql-16.4.5 +type: Opaque +data: + postgres-password: "WFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFg=" + password: "WFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFg=" + # We don't auto-generate LDAP password when it's not provided as we do for other passwords +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/config/secret.yaml +kind: Secret +apiVersion: v1 +metadata: + name: risk-1-secret + labels: + app: risk-1 + environment: ALPHA-1 + release: release-name +type: Opaque +stringData: + # secrets + AWS_S3_ACCESS_KEY_ID: "risk-module" + AWS_S3_BUCKET_NAME: "risk-module-storage" + AWS_S3_ENDPOINT_URL: "https://risk-minio-1.ifrcgo.local.example.com/" + AWS_S3_REGION: "us-east-1" + AWS_S3_SECRET_ACCESS_KEY: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + DATABASE_HOST: "ifrcgo-risk-module-pg" + DATABASE_NAME: "risk-module" + DATABASE_PASSWORD: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + DATABASE_PORT: "5432" + DATABASE_USER: "postgres" + DJANGO_SECRET_KEY: "xyz" + METEOSWISS_S3_ACCESS_KEY: "dummy-access-key" + METEOSWISS_S3_BUCKET: "dummy-bucket" + METEOSWISS_S3_ENDPOINT_URL: "https://dummy.endpoint" + METEOSWISS_S3_SECRET_KEY: "dummy-secret-key" + PDC_ACCESS_TOKEN: "dummy-token" + PDC_PASSWORD: "dummy-password" + PDC_USERNAME: "dummy-username" + SENTRY_DSN: "https://xyz@sentry.local.example.com/14" + USE_S3_BUCKET: "true" +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/provisioning-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: fircgo-risk-module-minio-provisioning + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 + app.kubernetes.io/component: minio-provisioning +data: +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/postgresql/templates/primary/initialization-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ifrcgo-risk-module-pg-init-scripts + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgresql + app.kubernetes.io/version: 17.2.0 + helm.sh/chart: postgresql-16.4.5 +data: + my_init_script.sh: | + #!/bin/sh -e + DATABASE_DUMP_URL="$DUMP_REGISTRY_URL$DUMP_FILENAME" + # Don't have curl here + /usr/lib/apt/apt-helper download-file -o "Acquire::https::Verify-Peer=false" "$DATABASE_DUMP_URL" /tmp/db.dump + du -sh /tmp/db.dump + PGPASSWORD="${POSTGRES_PASSWORD}" PGUSER=$POSTGRES_USER pg_restore -v --no-owner --no-privileges -d "$POSTGRES_DATABASE" < /tmp/db.dump +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/config/configmap.yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: risk-1-env-name + labels: + app: risk-1 + environment: ALPHA-1 + release: release-name +data: + # Configs + APPS_LOGGING_LEVEL: "WARNING" + DJANGO_ALLOWED_HOSTS: "*" + DJANGO_DEBUG: "false" + DJANGO_TIME_ZONE: "UTC" + RISK_API_FQDN: "risk-api-1.ifrcgo.local.example.com" + RISK_ENVIRONMENT: "ALPHA-1" + RISK_RELEASE: "develop.xxxxxxxx" + SENTRY_PROFILE_SAMPLE_RATE: "0.8" + SENTRY_TRACE_SAMPLE_RATE: "0.8" + TIME_ZONE: "UTC" +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/pvc.yaml +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: fircgo-risk-module-minio + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "0.5Gi" + storageClassName: local-path +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fircgo-risk-module-minio + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 +spec: + type: ClusterIP + ports: + - name: minio-api + port: 9000 + targetPort: minio-api + nodePort: null + - name: minio-console + port: 9001 + targetPort: minio-console + nodePort: null + selector: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: minio +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/postgresql/templates/primary/svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: ifrcgo-risk-module-pg-hl + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgresql + app.kubernetes.io/version: 17.2.0 + helm.sh/chart: postgresql-16.4.5 + app.kubernetes.io/component: primary + annotations: +spec: + type: ClusterIP + clusterIP: None + # We want all pods in the StatefulSet to have their addresses published for + # the sake of the other Postgresql pods even before they're ready, since they + # have to be able to talk to each other in order to become ready. + publishNotReadyAddresses: true + ports: + - name: tcp-postgresql + port: 5432 + targetPort: tcp-postgresql + selector: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql + app.kubernetes.io/component: primary +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/postgresql/templates/primary/svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: ifrcgo-risk-module-pg + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgresql + app.kubernetes.io/version: 17.2.0 + helm.sh/chart: postgresql-16.4.5 + app.kubernetes.io/component: primary +spec: + type: ClusterIP + sessionAffinity: None + ports: + - name: tcp-postgresql + port: 5432 + targetPort: tcp-postgresql + nodePort: null + selector: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql + app.kubernetes.io/component: primary +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/api/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: risk-1-api + labels: + app: risk-1 + component: api + environment: ALPHA-1 + release: release-name +spec: + type: ClusterIP + selector: + app: risk-1 + component: api + ports: + - protocol: TCP + port: 80 + targetPort: 80 +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/standalone/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fircgo-risk-module-minio + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 +spec: + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: minio + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 + annotations: + checksum/credentials-secret: d53dea05c271cd688f9b9ef0a04ebb5f9b883ab2ae43b121274a9c9b4d3f9934 + spec: + + serviceAccountName: fircgo-risk-module-minio + affinity: + podAffinity: + + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: minio + topologyKey: kubernetes.io/hostname + weight: 1 + nodeAffinity: + + nodeSelector: + kubernetes.io/hostname: desktop-01 + automountServiceAccountToken: false + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: OnRootMismatch + supplementalGroups: [] + sysctls: [] + containers: + - name: minio + image: docker.io/bitnamilegacy/minio:2024.12.18-debian-12-r1 + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + env: + - name: BITNAMI_DEBUG + value: "false" + - name: MINIO_SCHEME + value: "http" + - name: MINIO_FORCE_NEW_KEYS + value: "yes" + - name: MINIO_API_PORT_NUMBER + value: "9000" + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: fircgo-risk-module-minio + key: root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: fircgo-risk-module-minio + key: root-password + - name: MINIO_DEFAULT_BUCKETS + value: risk-module-storage + - name: MINIO_BROWSER + value: "off" + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: "public" + - name: MINIO_CONSOLE_PORT_NUMBER + value: "9001" + - name: MINIO_DATA_DIR + value: "/bitnami/minio/data" + envFrom: + ports: + - name: minio-api + containerPort: 9000 + protocol: TCP + - name: minio-console + containerPort: 9001 + protocol: TCP + livenessProbe: + httpGet: + path: /minio/health/live + port: minio-api + scheme: "HTTP" + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + tcpSocket: + port: minio-api + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + resources: + limits: + cpu: "4" + memory: 256Mi + requests: + cpu: "0.1" + memory: 128Mi + volumeMounts: + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + - name: empty-dir + mountPath: /opt/bitnami/minio/tmp + subPath: app-tmp-dir + - name: empty-dir + mountPath: /.mc + subPath: app-mc-dir + - name: data + mountPath: /bitnami/minio/data + volumes: + - name: empty-dir + emptyDir: {} + - name: data + persistentVolumeClaim: + claimName: fircgo-risk-module-minio +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/api/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: risk-1-api + annotations: + reloader.stakater.com/auto: "true" + labels: + app: risk-1 + component: api + environment: ALPHA-1 + release: release-name +spec: + replicas: 1 + selector: + matchLabels: + app: risk-1 + component: api + template: + metadata: + annotations: + checksum/secret: a5d73225a01c3e24b1530c338a0a44cf07ab77dc8646dc0562caada4bbc1b61f + checksum/configmap: f00be2c6cf51cc19c80358f167f85af68caa5a8c2405a310fc0fab05dfb0a0c8 + labels: + app: risk-1 + component: api + spec: + containers: + - name: api + image: "local.git.com/ifrcgo/risk:develop.xxxxxxxx" + imagePullPolicy: IfNotPresent + command: + - gunicorn + - risk_module.wsgi:application + - --bind + - 0.0.0.0:80 + ports: + - name: http + containerPort: 80 + protocol: TCP + # TODO: livenessProbe + resources: + limits: + cpu: "4" + memory: 2Gi + requests: + cpu: "0.1" + memory: 2Gi + envFrom: + - secretRef: + name: risk-1-secret + - configMapRef: + name: risk-1-env-name + env: + - name: DJANGO_APP_TYPE + value: "web" + - name: CACHE_REDIS_URL + value: redis://risk-module-dragonfly:6379/0 + - name: CELERY_REDIS_URL + value: redis://risk-module-dragonfly:6379/1 +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/worker-beat/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: risk-1-worker-beat + annotations: + reloader.stakater.com/auto: "true" + labels: + app: risk-1 + component: worker-beat + environment: ALPHA-1 + release: release-name +spec: + replicas: 1 # This should only 1 + selector: + matchLabels: + app: risk-1 + component: worker-beat + template: + metadata: + annotations: + checksum/secret: a5d73225a01c3e24b1530c338a0a44cf07ab77dc8646dc0562caada4bbc1b61f + checksum/configmap: f00be2c6cf51cc19c80358f167f85af68caa5a8c2405a310fc0fab05dfb0a0c8 + labels: + app: risk-1 + component: worker-beat + spec: + containers: + - name: worker-beat + image: "local.git.com/ifrcgo/risk:develop.xxxxxxxx" + imagePullPolicy: IfNotPresent + command: + - celery + - --app=risk_module + - beat + - --max-interval=3600 + - --loglevel=info + - --scheduler=django_celery_beat.schedulers:DatabaseScheduler + # TODO: livenessProbe + resources: + limits: + cpu: "4" + memory: 0.5Gi + requests: + cpu: "0.1" + memory: 0.2Gi + envFrom: + - secretRef: + name: risk-1-secret + - configMapRef: + name: risk-1-env-name + env: + - name: DJANGO_APP_TYPE + value: "worker" + - name: CACHE_REDIS_URL + value: redis://risk-module-dragonfly:6379/0 + - name: CELERY_REDIS_URL + value: redis://risk-module-dragonfly:6379/1 +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/worker/deployment.yaml +# Queue: default +apiVersion: apps/v1 +kind: Deployment +metadata: + name: risk-1-worker-default + annotations: + reloader.stakater.com/auto: "true" + labels: + app: risk-1 + component: worker + queue: default + environment: ALPHA-1 + release: release-name +spec: + replicas: 1 + selector: + matchLabels: + app: risk-1 + component: worker + queue: default + template: + metadata: + annotations: + checksum/secret: a5d73225a01c3e24b1530c338a0a44cf07ab77dc8646dc0562caada4bbc1b61f + checksum/configmap: f00be2c6cf51cc19c80358f167f85af68caa5a8c2405a310fc0fab05dfb0a0c8 + labels: + app: risk-1 + component: worker + queue: default + spec: + containers: + - name: worker + command: + - celery + - --app=risk_module + - worker + - --loglevel=INFO + - --queues=celery + - --max-tasks-per-child=20 + image: "local.git.com/ifrcgo/risk:develop.xxxxxxxx" + imagePullPolicy: IfNotPresent + # TODO: livenessProbe + resources: + limits: + cpu: "4" + memory: 4Gi + requests: + cpu: "0.1" + memory: 4Gi + envFrom: + - secretRef: + name: risk-1-secret + - configMapRef: + name: risk-1-env-name + env: + - name: DJANGO_APP_TYPE + value: "worker" + - name: CACHE_REDIS_URL + value: redis://risk-module-dragonfly:6379/0 + - name: CELERY_REDIS_URL + value: redis://risk-module-dragonfly:6379/1 +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/postgresql/templates/primary/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ifrcgo-risk-module-pg + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgresql + app.kubernetes.io/version: 17.2.0 + helm.sh/chart: postgresql-16.4.5 + app.kubernetes.io/component: primary +spec: + replicas: 1 + serviceName: ifrcgo-risk-module-pg-hl + updateStrategy: + rollingUpdate: {} + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql + app.kubernetes.io/component: primary + template: + metadata: + name: ifrcgo-risk-module-pg + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgresql + app.kubernetes.io/version: 17.2.0 + helm.sh/chart: postgresql-16.4.5 + app.kubernetes.io/component: primary + spec: + serviceAccountName: ifrcgo-risk-module-pg + + automountServiceAccountToken: false + affinity: + podAffinity: + + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql + app.kubernetes.io/component: primary + topologyKey: kubernetes.io/hostname + weight: 1 + nodeAffinity: + + nodeSelector: + kubernetes.io/hostname: desktop-01 + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + hostNetwork: false + hostIPC: false + containers: + - name: postgresql + image: docker.io/bitnamilegacy/postgresql:17.2.0-debian-12-r8 + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + env: + - name: BITNAMI_DEBUG + value: "false" + - name: POSTGRESQL_PORT_NUMBER + value: "5432" + - name: POSTGRESQL_VOLUME_DIR + value: "/bitnami/postgresql" + - name: PGDATA + value: "/bitnami/postgresql/data" + # Authentication + - name: POSTGRES_USER + value: "risk-module" + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: ifrcgo-risk-module-pg + key: password + - name: POSTGRES_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: ifrcgo-risk-module-pg + key: postgres-password + - name: POSTGRES_DATABASE + value: "risk-module" + # LDAP + - name: POSTGRESQL_ENABLE_LDAP + value: "no" + # TLS + - name: POSTGRESQL_ENABLE_TLS + value: "no" + # Audit + - name: POSTGRESQL_LOG_HOSTNAME + value: "false" + - name: POSTGRESQL_LOG_CONNECTIONS + value: "false" + - name: POSTGRESQL_LOG_DISCONNECTIONS + value: "false" + - name: POSTGRESQL_PGAUDIT_LOG_CATALOG + value: "off" + # Others + - name: POSTGRESQL_CLIENT_MIN_MESSAGES + value: "error" + - name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES + value: "pgaudit" + - name: DUMP_REGISTRY_URL + value: https://user:password@secure-dump.com + - name: DUMP_FILENAME + value: /risk/k8s-alpha-init-db.dump + ports: + - name: tcp-postgresql + containerPort: 5432 + livenessProbe: + failureThreshold: 6 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + exec: + command: + - /bin/sh + - -c + - exec pg_isready -U "risk-module" -d "dbname=risk-module" -h 127.0.0.1 -p 5432 + readinessProbe: + failureThreshold: 6 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + exec: + command: + - /bin/sh + - -c + - -e + - | + exec pg_isready -U "risk-module" -d "dbname=risk-module" -h 127.0.0.1 -p 5432 + resources: + limits: + cpu: "4" + requests: + cpu: "0.2" + memory: 1Gi + volumeMounts: + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + - name: empty-dir + mountPath: /opt/bitnami/postgresql/conf + subPath: app-conf-dir + - name: empty-dir + mountPath: /opt/bitnami/postgresql/tmp + subPath: app-tmp-dir + - name: custom-init-scripts + mountPath: /docker-entrypoint-initdb.d/ + - name: dshm + mountPath: /dev/shm + - name: data + mountPath: /bitnami/postgresql + volumes: + - name: empty-dir + emptyDir: {} + - name: custom-init-scripts + configMap: + name: ifrcgo-risk-module-pg-init-scripts + - name: dshm + emptyDir: + medium: Memory + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "2Gi" + storageClassName: local-path +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/argo-hooks/hook-job.yaml +# Hook: collect-static +apiVersion: batch/v1 +kind: Job +metadata: + name: risk-1-collect-static + annotations: + argocd.argoproj.io/hook: PostSync +spec: + template: + metadata: + annotations: + checksum/secret: a5d73225a01c3e24b1530c338a0a44cf07ab77dc8646dc0562caada4bbc1b61f + checksum/configmap: f00be2c6cf51cc19c80358f167f85af68caa5a8c2405a310fc0fab05dfb0a0c8 + labels: + app: risk-1 + component: argo-hooks + spec: + restartPolicy: "Never" + containers: + - name: collect-static + image: "local.git.com/ifrcgo/risk:develop.xxxxxxxx" + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -c + args: + - ./manage.py wait_for_resources --minio && ./manage.py collectstatic --noinput + resources: + limits: + cpu: "4" + memory: 1Gi + requests: + cpu: "0.1" + memory: 1Gi + envFrom: + - secretRef: + name: risk-1-secret + - configMapRef: + name: risk-1-env-name + env: + - name: DJANGO_APP_TYPE + value: "hook" + - name: CACHE_REDIS_URL + value: redis://risk-module-dragonfly:6379/0 + - name: CELERY_REDIS_URL + value: redis://risk-module-dragonfly:6379/1 +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/argo-hooks/hook-job.yaml +# Hook: create-users +apiVersion: batch/v1 +kind: Job +metadata: + name: risk-1-create-users + annotations: + argocd.argoproj.io/hook: PostSync +spec: + template: + metadata: + annotations: + checksum/secret: a5d73225a01c3e24b1530c338a0a44cf07ab77dc8646dc0562caada4bbc1b61f + checksum/configmap: f00be2c6cf51cc19c80358f167f85af68caa5a8c2405a310fc0fab05dfb0a0c8 + labels: + app: risk-1 + component: argo-hooks + spec: + restartPolicy: "Never" + containers: + - name: create-users + image: "local.git.com/ifrcgo/risk:develop.xxxxxxxx" + imagePullPolicy: IfNotPresent + command: + - ./manage.py + - shell + - -c + - | + import os; + from django.contrib.auth import get_user_model; + from django.db import transaction; + + User = get_user_model() + + user_credentials = [ + user_credential.split(":") + for user_credential in os.environ["ADMIN_USER_EMAIL_PASSWORD_SET"].split(",") + ]; + + print("Parsed provided credentials:", user_credentials); + + @transaction.atomic + def create_initial_users(user_credentials): + for first_name, last_name, username, email, password_hash in user_credentials: + user, created = User.objects.update_or_create( + email=email, + username=username, + defaults={ + "first_name": first_name, + "last_name": last_name, + "password": password_hash, + "is_superuser": True, + "is_staff": True, + }, + ) + action = "updated" + if created: + action = "created" + print(f"Successfully {action} user({email=})"); + create_initial_users(user_credentials); + resources: + limits: + cpu: "4" + memory: 1Gi + requests: + cpu: "0.1" + memory: 1Gi + envFrom: + - secretRef: + name: risk-1-secret + - configMapRef: + name: risk-1-env-name + env: + - name: DJANGO_APP_TYPE + value: "hook" + - name: CACHE_REDIS_URL + value: redis://risk-module-dragonfly:6379/0 + - name: CELERY_REDIS_URL + value: redis://risk-module-dragonfly:6379/1 + - name: ADMIN_USER_EMAIL_PASSWORD_SET + value: "IFRC:Admin:admin@ifrc.org:password_hash" +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/argo-hooks/hook-job.yaml +# Hook: db-migrate +apiVersion: batch/v1 +kind: Job +metadata: + generateName: risk-1-db-migrate- + annotations: + argocd.argoproj.io/hook: PostSync +spec: + template: + metadata: + annotations: + checksum/secret: a5d73225a01c3e24b1530c338a0a44cf07ab77dc8646dc0562caada4bbc1b61f + checksum/configmap: f00be2c6cf51cc19c80358f167f85af68caa5a8c2405a310fc0fab05dfb0a0c8 + labels: + app: risk-1 + component: argo-hooks + spec: + restartPolicy: "Never" + containers: + - name: db-migrate + image: "local.git.com/ifrcgo/risk:develop.xxxxxxxx" + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -c + args: + - ./manage.py wait_for_resources --db && ./manage.py migrate + resources: + limits: + cpu: "4" + memory: 1Gi + requests: + cpu: "0.1" + memory: 1Gi + envFrom: + - secretRef: + name: risk-1-secret + - configMapRef: + name: risk-1-env-name + env: + - name: DJANGO_APP_TYPE + value: "hook" + - name: CACHE_REDIS_URL + value: redis://risk-module-dragonfly:6379/0 + - name: CELERY_REDIS_URL + value: redis://risk-module-dragonfly:6379/1 +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/api-ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: fircgo-risk-module-minio-api + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: 500m +spec: + ingressClassName: "nginx" + rules: + - host: risk-minio-1.ifrcgo.local.example.com + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: fircgo-risk-module-minio + port: + name: minio-api +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/api/ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: risk-1-api + labels: + app: risk-1 + component: api + environment: ALPHA-1 + release: release-name +spec: + ingressClassName: "nginx" + rules: + - host: "risk-1.ifrcgo.local.example.com" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: risk-1-api + port: + number: 80 +--- +# Source: ifrcgo-risk-module-helm/charts/app/templates/extraManifests.yaml +# Manifest: dragonflydb-cluster +apiVersion: dragonflydb.io/v1alpha1 +kind: Dragonfly +metadata: + labels: + app.kubernetes.io/name: dragonfly + name: risk-module-dragonfly +spec: + args: + - --dbfilename=mainSnapshot + replicas: 1 + resources: + limits: + cpu: "1" + memory: 750Mi + requests: + cpu: "0.1" + memory: 500Mi + snapshot: + cron: '*/5 * * * *' + persistentVolumeClaimSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: local-path +--- +# Source: ifrcgo-risk-module-helm/charts/app/charts/minio/templates/provisioning-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: fircgo-risk-module-minio-provisioning + namespace: "default" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: minio + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 + app.kubernetes.io/component: minio-provisioning + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation +spec: + ttlSecondsAfterFinished: 600 + parallelism: 1 + template: + metadata: + labels: + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/version: 2024.12.18 + helm.sh/chart: minio-14.10.5 + app.kubernetes.io/component: minio-provisioning + spec: + + restartPolicy: OnFailure + terminationGracePeriodSeconds: 0 + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + serviceAccountName: fircgo-risk-module-minio + initContainers: + - name: wait-for-available-minio + image: docker.io/bitnami/os-shell:12-debian-12-r35 + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + command: + - /bin/bash + - -c + - |- + set -e; + echo "Waiting for Minio"; + wait-for-port \ + --host=fircgo-risk-module-minio \ + --state=inuse \ + --timeout=120 \ + 9000; + echo "Minio is available"; + resources: + limits: + cpu: 150m + ephemeral-storage: 2Gi + memory: 192Mi + requests: + cpu: 100m + ephemeral-storage: 50Mi + memory: 128Mi + containers: + - name: minio + image: docker.io/bitnamilegacy/minio:2024.12.18-debian-12-r1 + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + command: + - /bin/bash + - -c + - |- + set -e; + echo "Start Minio provisioning"; + + retry_while() { + local -r cmd="${1:?cmd is missing}" + local -r retries="${2:-12}" + local -r sleep_time="${3:-5}" + local return_value=1 + + read -r -a command <<< "$cmd" + for ((i = 1 ; i <= retries ; i+=1 )); do + "${command[@]}" && return_value=0 && break + sleep "$sleep_time" + done + return $return_value + } + + function attachPolicy() { + local tmp=$(mc admin $1 info provisioning $2 | sed -n -e 's/^Policy.*: \(.*\)$/\1/p'); + IFS=',' read -r -a CURRENT_POLICIES <<< "$tmp"; + if [[ ! "${CURRENT_POLICIES[*]}" =~ "$3" ]]; then + mc admin policy attach provisioning $3 --$1=$2; + fi; + }; + + function detachDanglingPolicies() { + local tmp=$(mc admin $1 info provisioning $2 | sed -n -e 's/^Policy.*: \(.*\)$/\1/p'); + IFS=',' read -r -a CURRENT_POLICIES <<< "$tmp"; + IFS=',' read -r -a DESIRED_POLICIES <<< "$3"; + for current in "${CURRENT_POLICIES[@]}"; do + if [[ ! "${DESIRED_POLICIES[*]}" =~ "${current}" ]]; then + mc admin policy detach provisioning $current --$1=$2; + fi; + done; + } + + function addUsersFromFile() { + local username=$(grep -oP '^username=\K.+' $1); + local password=$(grep -oP '^password=\K.+' $1); + local disabled=$(grep -oP '^disabled=\K.+' $1); + local policies_list=$(grep -oP '^policies=\K.+' $1); + local set_policies=$(grep -oP '^setPolicies=\K.+' $1); + + mc admin user add provisioning "${username}" "${password}"; + + IFS=',' read -r -a POLICIES <<< "${policies_list}"; + for policy in "${POLICIES[@]}"; do + attachPolicy user "${username}" "${policy}"; + done; + if [ "${set_policies}" == "true" ]; then + detachDanglingPolicies user "${username}" "${policies_list}"; + fi; + + local user_status="enable"; + if [[ "${disabled}" != "" && "${disabled,,}" == "true" ]]; then + user_status="disable"; + fi; + + mc admin user "${user_status}" provisioning "${username}"; + }; + mc alias set provisioning $MINIO_SCHEME://fircgo-risk-module-minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD; + + mc admin service restart provisioning --wait --json; + + # Adding a sleep to ensure that the check below does not cause + # a race condition. We check for the MinIO port because the + # "mc admin service restart --wait" command is not working as expected + sleep 5; + echo "Waiting for Minio to be available after restart"; + if ! retry_while "mc admin info provisioning"; then + echo "Error connecting to Minio" + exit 1 + fi + echo "Minio is available. Executing provisioning commands"; + + mc anonymous set download provisioning/risk-module-storage; + + echo "End Minio provisioning"; + env: + - name: MINIO_SCHEME + value: "http" + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: fircgo-risk-module-minio + key: root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: fircgo-risk-module-minio + key: root-password + envFrom: + resources: + limits: + cpu: 150m + ephemeral-storage: 2Gi + memory: 192Mi + requests: + cpu: 100m + ephemeral-storage: 50Mi + memory: 128Mi + volumeMounts: + - name: empty-dir + mountPath: /.mc + subPath: app-mc-dir + - name: empty-dir + mountPath: /opt/bitnami/minio/tmp + subPath: app-tmp-dir + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + - name: minio-provisioning + mountPath: /etc/ilm + volumes: + - name: empty-dir + emptyDir: {} + - name: minio-provisioning + configMap: + name: fircgo-risk-module-minio-provisioning diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl deleted file mode 100644 index 2f790b1..0000000 --- a/helm/templates/_helpers.tpl +++ /dev/null @@ -1,54 +0,0 @@ -{{/* - Expand the name of the chart. -*/}} -{{- define "ifrcgo-risk-module.name" -}} - {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* - Create a default fully qualified app name. - We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). - If release name contains chart name it will be used as a full name. - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names -*/}} -{{- define "ifrcgo-risk-module.fullname" -}} - {{- if .Values.fullnameOverride -}} - {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} - {{- else -}} - {{- $name := default .Chart.Name .Values.nameOverride -}} - {{- if contains $name .Release.Name -}} - {{- .Release.Name | trunc 63 | trimSuffix "-" -}} - {{- else -}} - {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} - {{- end -}} - {{- end -}} -{{- end -}} - -{{/* - Create chart name and version as used by the chart label. -*/}} -{{- define "ifrcgo-risk-module.chart" -}} - {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create the name of the service account to use -*/}} -{{- define "ifrcgo-risk-module.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "ifrcgo-risk-module.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* -Create the name of the secret to be used by the ifrcgo-risk-module -*/}} -{{- define "ifrcgo-risk-module.secretname" -}} -{{- if .Values.secretsName }} - {{- .Values.secretsName -}} -{{- else }} - {{- printf "%s-secret" (include "ifrcgo-risk-module.fullname" .) -}} -{{- end -}} -{{- end -}} diff --git a/helm/templates/api/deployment.yaml b/helm/templates/api/deployment.yaml deleted file mode 100644 index 3f82cdf..0000000 --- a/helm/templates/api/deployment.yaml +++ /dev/null @@ -1,73 +0,0 @@ -{{- if .Values.api.enabled }} - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "ifrcgo-risk-module.fullname" . }}-api - labels: - app.kubernetes.io/name: {{ include "ifrcgo-risk-module.fullname" . }} - app.kubernetes.io/component: {{ include "ifrcgo-risk-module.fullname" . }}-api - environment: {{ .Values.environment }} - release: {{ .Release.Name }} - annotations: - reloader.stakater.com/auto: "true" -spec: - replicas: {{ .Values.api.replicaCount }} - selector: - matchLabels: - app.kubernetes.io/name: {{ include "ifrcgo-risk-module.fullname" . }} - app.kubernetes.io/component: {{ include "ifrcgo-risk-module.fullname" . }}-api - template: - metadata: - {{- if not .Values.azure.aksSecretsProviderAvailable }} - annotations: - checksum/secret: {{ include (print .Template.BasePath "/config/secret.yaml") . | sha256sum }} - checksum/configmap: {{ include (print .Template.BasePath "/config/configmap.yaml") . | sha256sum }} - {{- end }} - labels: - app.kubernetes.io/name: {{ include "ifrcgo-risk-module.fullname" . }} - app.kubernetes.io/component: {{ include "ifrcgo-risk-module.fullname" . }}-api - spec: - containers: - - name: {{ include "ifrcgo-risk-module.fullname" . }}-api - image: "{{ .Values.image.name }}:{{ .Values.image.tag }}" - command: ["bash", "/code/scripts/run_prod.sh"] - ports: - - name: http - containerPort: {{ .Values.api.containerPort }} - protocol: TCP - # livenessProbe: # TODO: Add Liveness Probe - envFrom: - - secretRef: - name: {{ template "ifrcgo-risk-module.secretname" . }} - - configMapRef: - name: {{ template "ifrcgo-risk-module.fullname" . }}-api-configmap - resources: - {{- toYaml .Values.api.resources | nindent 12 }} - {{- if .Values.azure.aksSecretsProviderAvailable }} - volumeMounts: - - name: {{ include "ifrcgo-risk-module.secretname" . }} - mountPath: /mnt/secrets-store - readOnly: true - {{- end }} - - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - - {{- if .Values.serviceAccount.create }} - serviceAccountName: {{ include "ifrcgo-risk-module.serviceAccountName" . }} - {{- end }} - - {{- if .Values.azure.aksSecretsProviderAvailable }} - volumes: - - name: {{ include "ifrcgo-risk-module.secretname" . }} - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: {{ include "ifrcgo-risk-module.fullname" . }}-secret-provider - {{- end }} - -{{- end }} diff --git a/helm/templates/api/ingress.yaml b/helm/templates/api/ingress.yaml deleted file mode 100644 index c7abf55..0000000 --- a/helm/templates/api/ingress.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if .Values.ingress.enabled }} - -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ include "ifrcgo-risk-module.fullname" . }}-ingress - labels: - app.kubernetes.io/name: {{ include "ifrcgo-risk-module.fullname" . }} - app.kubernetes.io/component: {{ include "ifrcgo-risk-module.fullname" . }}-api - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -spec: - ingressClassName: {{ required "ingress.className" .Values.ingress.className | quote }} - rules: - - host: {{ required "ingress.host" .Values.ingress.host | quote }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{ include "ifrcgo-risk-module.fullname" . }}-api-svc - port: - number: 80 - -{{- end }} diff --git a/helm/templates/api/service.yaml b/helm/templates/api/service.yaml deleted file mode 100644 index 29f1ab1..0000000 --- a/helm/templates/api/service.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if .Values.api.enabled -}} - -apiVersion: v1 -kind: Service -metadata: - name: {{ template "ifrcgo-risk-module.fullname" . }}-api-svc - labels: - app.kubernetes.io/name: {{ include "ifrcgo-risk-module.fullname" . }} - app.kubernetes.io/component: {{ include "ifrcgo-risk-module.fullname" . }}-api - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -spec: -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: {{ include "ifrcgo-risk-module.fullname" . }} - app.kubernetes.io/component: {{ include "ifrcgo-risk-module.fullname" . }}-api - ports: - - protocol: TCP - port: 80 - targetPort: {{ .Values.api.containerPort }} - -{{- end }} diff --git a/helm/templates/api/service_account.yaml b/helm/templates/api/service_account.yaml deleted file mode 100644 index c44cf61..0000000 --- a/helm/templates/api/service_account.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.serviceAccount.create -}} - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "ifrcgo-risk-module.serviceAccountName" . }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{ toYaml . | nindent 4}} - {{- end }} - labels: - azure.workload.identity/use: "true" -automountServiceAccountToken: {{ .Values.serviceAccount.automount }} - -{{- end }} diff --git a/helm/templates/config/configmap.yaml b/helm/templates/config/configmap.yaml deleted file mode 100644 index c9a5294..0000000 --- a/helm/templates/config/configmap.yaml +++ /dev/null @@ -1,31 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: {{ include "ifrcgo-risk-module.fullname" .}}-api-configmap - labels: - app.kubernetes.io/name: {{ include "ifrcgo-risk-module.fullname" . }} - app.kubernetes.io/component: {{ include "ifrcgo-risk-module.fullname" . }}-api - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -data: - # Application info - RISK_RELEASE: {{ .Values.image.tag | quote }} - RISK_ENVIRONMENT: {{ .Values.environment | quote | upper }} - RISK_API_FQDN: {{ required "env.RISK_API_FQDN" .Values.env.RISK_API_FQDN | quote }} - # Django configs - DJANGO_DEBUG: {{ .Values.env.DJANGO_DEBUG | quote }} - DJANGO_ALLOWED_HOSTS: {{ .Values.env.DJANGO_ALLOWED_HOSTS | quote }} - TIME_ZONE: {{ .Values.env.TIME_ZONE | quote }} - # Redis - {{- if .Values.redis.enabled }} - CELERY_REDIS_URL: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/0" - CACHE_REDIS_URL: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/1" - {{- else }} - CELERY_REDIS_URL: {{ required "env.CELERY_REDIS_URL" .Values.env.CELERY_REDIS_URL | quote }} - CACHE_REDIS_URL: {{ required "env.CACHE_REDIS_URL" .Values.env.CACHE_REDIS_URL | quote }} - {{- end }} - # Sentry - SENTRY_TRACE_SAMPLE_RATE: {{ .Values.env.SENTRY_TRACE_SAMPLE_RATE | quote }} - SENTRY_PROFILE_SAMPLE_RATE: {{ .Values.env.SENTRY_PROFILE_SAMPLE_RATE | quote }} - # Logging - APPS_LOGGING_LEVEL: {{ .Values.env.APPS_LOGGING_LEVEL | quote }} diff --git a/helm/templates/config/secret.yaml b/helm/templates/config/secret.yaml deleted file mode 100644 index 5b40ed9..0000000 --- a/helm/templates/config/secret.yaml +++ /dev/null @@ -1,43 +0,0 @@ -{{- if not .Values.azure.aksSecretsProviderAvailable -}} - -kind: Secret -apiVersion: v1 -metadata: - name: {{ template "ifrcgo-risk-module.secretname" . }} - labels: - component: api-deployment - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -type: Opaque -stringData: - DJANGO_SECRET_KEY: {{ required "secrets.DJANGO_SECRET_KEY" .Values.secrets.DJANGO_SECRET_KEY | quote }} - - # Database - {{- if .Values.postgresql.enabled }} - DATABASE_NAME: {{ .Values.postgresql.auth.database | quote }} - DATABASE_USER: {{ .Values.postgresql.auth.username | quote }} - DATABASE_PASSWORD: {{ .Values.postgresql.auth.password | quote }} - DATABASE_HOST: {{ .Values.postgresql.auth.password | quote }} - DATABASE_HOST: {{ include "postgresql.v1.primary.fullname" .Subcharts.postgresql | quote }} - DATABASE_PORT: {{ include "postgresql.v1.service.port" .Subcharts.postgresql | quote }} - {{- else }} - DATABASE_NAME: {{ required "secrets.DATABASE_NAME" .Values.secrets.DATABASE_NAME | quote }} - DATABASE_USER: {{ required "secrets.DATABASE_USER" .Values.secrets.DATABASE_USER | quote }} - DATABASE_PASSWORD: {{ required "secrets.DATABASE_PASSWORD" .Values.secrets.DATABASE_PASSWORD | quote }} - DATABASE_HOST: {{ required "secrets.DATABASE_HOST" .Values.secrets.DATABASE_HOST | quote }} - DATABASE_PORT: {{ required "secrets.DATABASE_PORT" .Values.secrets.DATABASE_PORT | quote }} - {{- end }} - - # Sentry - SENTRY_DSN: {{ required "secrets.SENTRY_DSN" .Values.secrets.SENTRY_DSN | quote }} - # PDC - PDC_USERNAME: {{ required "secrets.PDC_USERNAME" .Values.secrets.PDC_USERNAME | quote }} - PDC_PASSWORD: {{ required "secrets.PDC_PASSWORD" .Values.secrets.PDC_PASSWORD | quote }} - PDC_ACCESS_TOKEN: {{ required "secrets.PDC_ACCESS_TOKEN" .Values.secrets.PDC_ACCESS_TOKEN | quote }} - # Meteoswiss - METEOSWISS_S3_ENDPOINT_URL: {{ required "secrets.METEOSWISS_S3_ENDPOINT_URL" .Values.secrets.METEOSWISS_S3_ENDPOINT_URL | quote }} - METEOSWISS_S3_BUCKET: {{ required "secrets.METEOSWISS_S3_BUCKET" .Values.secrets.METEOSWISS_S3_BUCKET | quote }} - METEOSWISS_S3_ACCESS_KEY: {{ required "secrets.METEOSWISS_S3_ACCESS_KEY" .Values.secrets.METEOSWISS_S3_ACCESS_KEY | quote }} - METEOSWISS_S3_SECRET_KEY: {{ required "secrets.METEOSWISS_S3_SECRET_KEY" .Values.secrets.METEOSWISS_S3_SECRET_KEY | quote }} - -{{- end }} diff --git a/helm/templates/config/secrets_provider_class.yaml b/helm/templates/config/secrets_provider_class.yaml deleted file mode 100644 index 7bec4f8..0000000 --- a/helm/templates/config/secrets_provider_class.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{- if .Values.azure.aksSecretsProviderAvailable -}} - -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: {{ include "ifrcgo-risk-module.fullname" . }}-secret-provider -spec: - provider: azure - parameters: - usePodIdentity: "false" - clientID: {{ .Values.azure.keyvault.clientId }} - keyvaultName: {{ .Values.azure.keyvault.name }} - tenantId: {{ .Values.azure.keyvault.tenantId }} - objects: | - array: - {{- range $name, $value := .Values.secrets }} - - | - objectName: {{ $name | upper | replace "_" "-" }} - objectType: secret - {{- end }} - secretObjects: - - secretName: {{ include "ifrcgo-risk-module.secretname" . }} - type: Opaque - data: - {{- range $name, $value := .Values.secrets }} - - objectName: {{ $name | replace "_" "-" | upper }} - key: {{ $name }} - {{- end }} - -{{- end }} diff --git a/helm/templates/cronjobs/jobs.yaml b/helm/templates/cronjobs/jobs.yaml deleted file mode 100644 index 7182be3..0000000 --- a/helm/templates/cronjobs/jobs.yaml +++ /dev/null @@ -1,46 +0,0 @@ -{{- if .Values.worker.enabled }} - -{{- range $jobName, $job := .Values.worker.cronjobs }} - -{{- if $job.enabled }} - ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: "{{ template "ifrcgo-risk-module.fullname" $ }}-{{ $jobName | nospace | lower | replace "_" "-" }}" - labels: - app.kubernetes.io/name: {{ include "ifrcgo-risk-module.fullname" $ }} - app.kubernetes.io/component: {{ include "ifrcgo-risk-module.fullname" $ }}-api-jobs - environment: {{ $.Values.environment }} - release: {{ $.Release.Name }} -spec: - schedule: {{ $job.schedule | quote }} - concurrencyPolicy: "Forbid" - jobTemplate: - spec: - activeDeadlineSeconds: {{ default 7200 $job.timeLimit }} # 2 hours default TODO: Lower this - template: - spec: - restartPolicy: "Never" - containers: - - name: "{{ template "ifrcgo-risk-module.fullname" $ }}-{{ $jobName | nospace | lower | replace "_" "-" }}" - image: "{{ $.Values.image.name }}:{{ $.Values.image.tag }}" - command: ["./manage.py", "{{ $jobName }}"] - resources: - {{- toYaml (default $.Values.worker.defaultResources $job.resources) | nindent 16 }} - envFrom: - - secretRef: - name: {{ include "ifrcgo-risk-module.secretname" $ }} - - configMapRef: - name: {{ template "ifrcgo-risk-module.fullname" $ }}-api-configmap - {{- with $.Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 12 }} - {{- end }} - -{{- end }} - -{{- end }} - -{{- end }} diff --git a/helm/tests.yaml b/helm/tests.yaml new file mode 100644 index 0000000..f1e9d9f --- /dev/null +++ b/helm/tests.yaml @@ -0,0 +1,4 @@ +tests: + alpha-1.yaml: + - values/operators.yaml + - values/alpha.yaml diff --git a/helm/tests/alpha-1.yaml b/helm/tests/alpha-1.yaml new file mode 100644 index 0000000..295b74c --- /dev/null +++ b/helm/tests/alpha-1.yaml @@ -0,0 +1,72 @@ +app: + environment: ALPHA-1 + fullnameOverride: risk-1 + + image: + name: local.git.com/ifrcgo/risk + tag: develop.xxxxxxxx + + ingress: + host: risk-1.ifrcgo.local.example.com + + postgresql: + auth: + postgresPassword: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + password: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + primary: + nodeSelector: + kubernetes.io/hostname: "desktop-01" + persistence: + storageClass: "local-path" + size: 2Gi + extraEnvVars: + - name: DUMP_REGISTRY_URL + value: https://user:password@secure-dump.com + - name: DUMP_FILENAME + value: /risk/k8s-alpha-init-db.dump + + minio: + nodeSelector: + kubernetes.io/hostname: "desktop-01" + global: + defaultStorageClass: "local-path" + apiIngress: + hostname: risk-minio-1.ifrcgo.local.example.com + auth: + rootPassword: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + persistence: + size: 0.5Gi + + env: + # App Domain + RISK_API_FQDN: "risk-api-1.ifrcgo.local.example.com" + + secrets: + DJANGO_SECRET_KEY: xyz + SENTRY_DSN: https://xyz@sentry.local.example.com/14 + # PDC + PDC_USERNAME: "dummy-username" + PDC_PASSWORD: "dummy-password" + PDC_ACCESS_TOKEN: "dummy-token" + # Meteoswiss + METEOSWISS_S3_ENDPOINT_URL: "https://dummy.endpoint" + METEOSWISS_S3_BUCKET: "dummy-bucket" + METEOSWISS_S3_ACCESS_KEY: "dummy-access-key" + METEOSWISS_S3_SECRET_KEY: "dummy-secret-key" + + argoHook: + enabled: true + hooks: + # NOTE: Make sure keys are lowercase + create-users: + enabled: true + hook: PostSync + env: + ADMIN_USER_EMAIL_PASSWORD_SET: "IFRC:Admin:admin@ifrc.org:password_hash" + + extraManifests: + dragonflydb-cluster: + spec: + snapshot: + persistentVolumeClaimSpec: + storageClassName: local-path diff --git a/helm/update-snapshots.sh b/helm/update-snapshots.sh new file mode 100755 index 0000000..ca171f0 --- /dev/null +++ b/helm/update-snapshots.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export SCRIPT_DIR + +# shellcheck disable=SC2068 +"$SCRIPT_DIR/../fugit/scripts/helm-update-snapshots.sh" $@ diff --git a/helm/values-test.yaml b/helm/values-test.yaml deleted file mode 100644 index efe7d8d..0000000 --- a/helm/values-test.yaml +++ /dev/null @@ -1,72 +0,0 @@ -environment: test - -fullnameOverride: ifrcgo-risk - -image: - name: ghcr.io/ifrcgo/go-risk-module-api - tag: test - -imagePullSecrets: - - name: ghcr-secret - -ingress: - enabled: true - host: go-risk-test.ifrc.org - -postgresql: - enabled: true - -redis: - enabled: true - -env: - # Application info - RISK_API_FQDN: go-risk-test.ifrc.org - # Django configs - DJANGO_DEBUG: false - DJANGO_ALLOWED_HOSTS: "*" - TIME_ZONE: "UTC" - # Sentry - SENTRY_TRACE_SAMPLE_RATE: "0.2" - SENTRY_PROFILE_SAMPLE_RATE: "0.2" - # Logging - APPS_LOGGING_LEVEL: "WARNING" - -secrets: - DJANGO_SECRET_KEY: test - # Database - DATABASE_NAME: test - DATABASE_USER: test - DATABASE_PASSWORD: test - DATABASE_HOST: test - DATABASE_PORT: 5432 - # Sentry - SENTRY_DSN: "https://test.com/test" - # PDC - PDC_USERNAME: test - PDC_PASSWORD: test - PDC_ACCESS_TOKEN: test - # Meteoswiss - METEOSWISS_S3_ENDPOINT_URL: test - METEOSWISS_S3_BUCKET: test - METEOSWISS_S3_ACCESS_KEY: test - METEOSWISS_S3_SECRET_KEY: test - - -azure: - aksSecretsProviderAvailable: true - keyvault: - name: risk-module-playground-k - clientId: "1d64d996-002b-47c0-ad1f-b2a400d83d5c" - tenantId: "a2b53be5-734e-4e6c-ab0d-d184f60fd917" - -serviceAccount: - # Specifies whether a service account should be created - # Required when using workload identity to access Azure Key Vault Secrets - create: true - automount: true - annotations: - azure.workload.identity/client-id : "1d64d996-002b-47c0-ad1f-b2a400d83d5c" - labels: - azure.workload.identity/use: "true" - name: "service-token-reader" diff --git a/helm/values.yaml b/helm/values.yaml index 8a81e60..fe155a4 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,167 +1,159 @@ -environment: prod +app: + environment: PROD + nameOverride: ifrcgo-risk-module -image: - name: SET-BY-CICD-IMAGE - tag: SET-BY-CICD-TAG + appTypeEnvName: "DJANGO_APP_TYPE" -imagePullSecrets: [] + # Global Image + image: + name: SET-BY-CICD-IMAGE + tag: SET-BY-CICD-TAG + imagePullPolicy: IfNotPresent -ingress: - enabled: false - className: webapprouting.kubernetes.azure.com - host: + ingress: + enabled: true + className: + host: + rabbitmq: + enabled: false -# Subchart configuration -# XXX: This is for local/playground usages only. -# For production use azure postgres server and provide credentials using azure keyvault -postgresql: - enabled: false - fullnameOverride: ifrcgo-risk-module-postgresql - architecture: standalone - auth: - postgresPassword: "postgres" - username: "postgres" - password: "postgres" - database: "risk" - primary: - persistence: - enabled: true - storageClass: "" - size: 8Gi + # XXX: This is for local/playground usages only. + postgresql: + enabled: false + fullnameOverride: ifrcgo-risk-module-pg + architecture: standalone + auth: + postgresPassword: "" + username: "" + password: "" + database: "" + primary: + persistence: + enabled: true + storageClass: "" + size: 8Gi -redis: - enabled: false - architecture: standalone - fullnameOverride: ifrcgo-risk-module-redis - auth: + redis: enabled: false - master: - persistence: - enabled: true - storageClass: "" - size: .5Gi + architecture: standalone + fullnameOverride: ifrcgo-risk-module-redis + auth: + enabled: false + master: + persistence: + enabled: true + storageClass: "" + size: .5Gi -api: - enabled: true - replicaCount: 1 - containerPort: 80 - resources: - requests: - cpu: "2" - memory: 0.5Gi - limits: - cpu: "2" - memory: 1Gi + minio: + enabled: false -env: - # Application info - # RISK_ENVIRONMENT: .environment is used - RISK_API_FQDN: - # Django configs - DJANGO_DEBUG: false - DJANGO_ALLOWED_HOSTS: "*" - TIME_ZONE: "UTC" - # Sentry - SENTRY_TRACE_SAMPLE_RATE: "0.2" - SENTRY_PROFILE_SAMPLE_RATE: "0.2" - # Logging - APPS_LOGGING_LEVEL: "WARNING" + api: + enabled: true + replicaCount: 1 + containerPort: 80 + command: ["gunicorn", "risk_module.wsgi:application", "--bind", "0.0.0.0:80"] + resources: + requests: + cpu: "0.1" + memory: 1Gi + limits: + cpu: "4" + memory: 2Gi -secretsName: "" -secrets: - DJANGO_SECRET_KEY: - # Database - DATABASE_NAME: - DATABASE_USER: - DATABASE_PASSWORD: - DATABASE_HOST: - DATABASE_PORT: 5432 - # Sentry - SENTRY_DSN: "" - # PDC - PDC_USERNAME: - PDC_PASSWORD: - PDC_ACCESS_TOKEN: - # Meteoswiss - METEOSWISS_S3_ENDPOINT_URL: - METEOSWISS_S3_BUCKET: - METEOSWISS_S3_ACCESS_KEY: - METEOSWISS_S3_SECRET_KEY: + worker: + enabled: true + flower: + enabled: false + beat: + command: + - celery + - --app=risk_module + - beat + - --max-interval=3600 + - --loglevel=info + - --scheduler=django_celery_beat.schedulers:DatabaseScheduler + resources: + requests: + cpu: "0.1" + memory: 0.2Gi + limits: + cpu: "1" + memory: 0.5Gi + queueDefaultResources: + requests: + cpu: "0.1" + memory: 1Gi + limits: + cpu: "4" + memory: 2Gi + queueCommandPrefix: + - celery + - --app=risk_module + - worker + - --loglevel=INFO + queues: + # NOTE: Make sure keys are lowercase + default: + enabled: true + replicaCount: 1 + celeryArgs: ["--queues=celery", "--max-tasks-per-child=20"] + resources: + requests: + memory: 4Gi + limits: + memory: 4Gi -worker: - enabled: true - defaultResources: - requests: - cpu: "1" - memory: 1Gi - limits: - cpu: "1" - memory: 2Gi - # NOTE: Application level configuration - cronjobs: - "import_earthquake_data": - schedule: "0 0 * * *" - enabled: true - # timeLimit: 7200 - "create_pdc_data": - schedule: "0 */2 * * *" - enabled: true - "create_pdc_daily": - schedule: "0 5 * * *" - enabled: true - "create_pdc_displacement": - schedule: "0 */3 * * *" - enabled: true - "create_pdc_polygon": - schedule: "0 */6 * * *" - enabled: true - "create_pdc_intensity": - schedule: "0 */7 * * *" - enabled: true - "check_pdc_status": - schedule: "0 1 * * *" - enabled: true - "create_hazard_information": - schedule: "0 0 2 * *" - enabled: true - "create_adam_exposure": - schedule: "0 */4 * * *" - enabled: true - "update_adam_cyclone": - schedule: "0 */4 * * *" - enabled: true - "update_adam_alert_level": - schedule: "0 */4 * * *" - enabled: true - "import_gdacs_data": - schedule: "0 */4 * * *" - enabled: true - "pull_meteoswiss": - schedule: "0 */4 * * *" - enabled: true - "pull_meteoswiss_geo": - schedule: "0 */4 * * *" - enabled: true - "meteoswiss_agg": - schedule: "0 */4 * * *" - enabled: true + argoHook: + enabled: true + resources: + requests: + cpu: "0.1" + memory: 1Gi + limits: + cpu: "4" + memory: 1Gi + hooks: + # NOTE: Make sure keys are lowercase + db-migrate: + enabled: true + hook: PostSync + preserveHistory: true + command: ["/bin/bash", "-c"] + args: + - "./manage.py wait_for_resources --db && ./manage.py migrate" + collect-static: + enabled: true + hook: PostSync + command: ["./manage.py", "collectstatic", "--noinput"] -# Azure configurations -azure: - aksSecretsProviderAvailable: false - keyvault: - name: "" - clientId: "" - tenantId: "" + env: + # Application info + RISK_RELEASE: "{{ $.Values.image.tag }}" + RISK_ENVIRONMENT: "{{ $.Values.environment }}" + RISK_API_FQDN: "{{ required \".app.secrets.EOAPI_STAC_API\" nil }}" + # Django configs + DJANGO_DEBUG: "false" + DJANGO_TIME_ZONE: UTC + DJANGO_ALLOWED_HOSTS: "*" + TIME_ZONE: "UTC" + # Sentry + SENTRY_TRACE_SAMPLE_RATE: "0.2" + SENTRY_PROFILE_SAMPLE_RATE: "0.2" + # Logging + APPS_LOGGING_LEVEL: "WARNING" -serviceAccount: - # Specifies whether a service account should be created - # Required when using workload identity to access Azure Key Vault Secrets - create: false - automount: true - annotations: - azure.workload.identity/client-id : "" - labels: - azure.workload.identity/use: "true" - name: "" + secrets: + DJANGO_SECRET_KEY: "{{ required \".app.secrets.DJANGO_SECRET_KEY\" nil }}" + # Sentry + SENTRY_DSN: "{{ required \".app.secrets.SENTRY_DSN\" nil }}" + # PDC + PDC_USERNAME: "{{ required \".app.secrets.PDC_USERNAME\" nil }}" + PDC_PASSWORD: "{{ required \".app.secrets.PDC_PASSWORD\" nil }}" + PDC_ACCESS_TOKEN: "{{ required \".app.secrets.PDC_ACCESS_TOKEN\" nil }}" + # Meteoswiss + METEOSWISS_S3_ENDPOINT_URL: "{{ required \".app.secrets.METEOSWISS_S3_ENDPOINT_URL\" nil }}" + METEOSWISS_S3_BUCKET: "{{ required \".app.secrets.METEOSWISS_S3_BUCKET\" nil }}" + METEOSWISS_S3_ACCESS_KEY: "{{ required \".app.secrets.METEOSWISS_S3_ACCESS_KEY\" nil }}" + METEOSWISS_S3_SECRET_KEY: "{{ required \".app.secrets.METEOSWISS_S3_SECRET_KEY\" nil }}" diff --git a/helm/values/alpha.yaml b/helm/values/alpha.yaml new file mode 100644 index 0000000..951bafb --- /dev/null +++ b/helm/values/alpha.yaml @@ -0,0 +1,154 @@ +app: + ingress: + enabled: true + className: nginx + + postgresql: + enabled: true + auth: + username: "risk-module" + database: "risk-module" + primary: + resources: + requests: + cpu: "0.2" + memory: 1Gi + limits: + cpu: "4" + initdb: + scripts: + my_init_script.sh: | + #!/bin/sh -e + DATABASE_DUMP_URL="$DUMP_REGISTRY_URL$DUMP_FILENAME" + # Don't have curl here + /usr/lib/apt/apt-helper download-file -o "Acquire::https::Verify-Peer=false" "$DATABASE_DUMP_URL" /tmp/db.dump + du -sh /tmp/db.dump + PGPASSWORD="${POSTGRES_PASSWORD}" PGUSER=$POSTGRES_USER pg_restore -v --no-owner --no-privileges -d "$POSTGRES_DATABASE" < /tmp/db.dump + + minio: + enabled: true + disableWebUI: true + mode: standalone + fullnameOverride: fircgo-risk-module-minio + apiIngress: + enabled: true + ingressClassName: nginx + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: "500m" + auth: + forceNewKeys: True + rootUser: risk-module + rootPassword: + resources: + requests: + cpu: "0.1" + memory: 128Mi + limits: + cpu: "4" + memory: 256Mi + persistence: + enabled: true + size: 500Mi + defaultBuckets: risk-module-storage + provisioning: + enabled: true + resourcesPreset: "nano" + cleanupAfterFinished: + enabled: true + extraCommands: + - "mc anonymous set download provisioning/risk-module-storage" + + + api: + resources: + requests: + cpu: "0.1" + memory: "2Gi" + limits: + cpu: "4" + memory: "2Gi" + + worker: + enabled: true + beat: + resources: + limits: + cpu: "4" + queueDefaultResources: + requests: + cpu: "0.1" + memory: 1Gi + limits: + cpu: "4" + memory: 2Gi + + env: + # Sentry + SENTRY_TRACE_SAMPLE_RATE: "0.8" + SENTRY_PROFILE_SAMPLE_RATE: "0.8" + + secrets: + # Database + DATABASE_NAME: "{{ $.Values.postgresql.auth.database }}" + DATABASE_USER: "postgres" + DATABASE_PASSWORD: "{{ $.Values.postgresql.auth.postgresPassword }}" + DATABASE_HOST: "{{ include \"postgresql.v1.primary.fullname\" $.Subcharts.postgresql }}" + DATABASE_PORT: "{{ include \"postgresql.v1.service.port\" $.Subcharts.postgresql }}" + # Minio + USE_S3_BUCKET: "true" + AWS_S3_ENDPOINT_URL: "https://{{ .Values.minio.apiIngress.hostname }}/" + AWS_S3_ACCESS_KEY_ID: "{{ required \".Values.minio.auth.rootUser\" .Values.minio.auth.rootUser }}" + AWS_S3_SECRET_ACCESS_KEY: "{{ required \".Values.minio.auth.rootPassword\" .Values.minio.auth.rootPassword }}" + AWS_S3_BUCKET_NAME: "risk-module-storage" + AWS_S3_REGION: "us-east-1" + + argoHook: + enabled: true + hooks: + # NOTE: Make sure keys are lowercase + collect-static: + command: ["/bin/bash", "-c"] + args: + - "./manage.py wait_for_resources --minio && ./manage.py collectstatic --noinput" + create-users: + enabled: true + # env: + # ADMIN_USER_EMAIL_PASSWORD_SET: "IFRC:Admin:ifrc-admin:admin@ifrc.org:password_hash" + hook: PostSync + command: + - ./manage.py + - shell + - -c + - | + import os; + from django.contrib.auth import get_user_model; + from django.db import transaction; + + User = get_user_model() + + user_credentials = [ + user_credential.split(":") + for user_credential in os.environ["ADMIN_USER_EMAIL_PASSWORD_SET"].split(",") + ]; + + print("Parsed provided credentials:", user_credentials); + + @transaction.atomic + def create_initial_users(user_credentials): + for first_name, last_name, username, email, password_hash in user_credentials: + user, created = User.objects.update_or_create( + email=email, + username=username, + defaults={ + "first_name": first_name, + "last_name": last_name, + "password": password_hash, + "is_superuser": True, + "is_staff": True, + }, + ) + action = "updated" + if created: + action = "created" + print(f"Successfully {action} user({email=})"); + create_initial_users(user_credentials); diff --git a/helm/values/operators.yaml b/helm/values/operators.yaml new file mode 100644 index 0000000..814782c --- /dev/null +++ b/helm/values/operators.yaml @@ -0,0 +1,35 @@ +app: + extraEnvVars: + # Checkout the resources defined by extraManifests + - name: CACHE_REDIS_URL + value: redis://risk-module-dragonfly:6379/0 + - name: CELERY_REDIS_URL + value: redis://risk-module-dragonfly:6379/1 + + extraManifests: + dragonflydb-cluster: + apiVersion: dragonflydb.io/v1alpha1 + kind: Dragonfly + metadata: + labels: + app.kubernetes.io/name: dragonfly + name: risk-module-dragonfly # NOTE: this will be the service name + spec: + replicas: 1 + args: ["--dbfilename=mainSnapshot"] # NOTE: Only maintains 1 last snapshot + resources: + requests: + cpu: "0.1" + memory: 500Mi + limits: + cpu: "1" + memory: 750Mi + snapshot: + cron: "*/5 * * * *" + persistentVolumeClaimSpec: + # storageClassName: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/prod-docker-compose.yml b/prod-docker-compose.yml index 49219b9..cca88a4 100644 --- a/prod-docker-compose.yml +++ b/prod-docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - x-server: &base_server_setup image: risk-module:latest build: diff --git a/pyproject.toml b/pyproject.toml index 76a1f58..a387e63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "django-celery-beat==2.2.1", "django-cors-headers==3.7.0", "django-crispy-forms==1.12.0", - "django-storages==1.11.1", + "django-storages[azure]~=1.11.1", "django-enumfield==2.0.2", "django-filter==2.4.0", "django-redis==5.0.0", @@ -39,6 +39,7 @@ dependencies = [ "python-Levenshtein==0.21.1", "django-health-check", "psutil", + "azure-identity", ] [dependency-groups] diff --git a/risk_module/settings.py b/risk_module/settings.py index 81492af..f3f626c 100644 --- a/risk_module/settings.py +++ b/risk_module/settings.py @@ -10,15 +10,18 @@ https://docs.djangoproject.com/en/3.2/ref/settings/ """ -import os from pathlib import Path import environ +from azure.identity import DefaultAzureCredential from celery.schedules import crontab from django.utils.log import DEFAULT_LOGGING from risk_module import sentry +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + env = environ.Env( # Application info RISK_ENVIRONMENT=str, @@ -35,12 +38,28 @@ DATABASE_PASSWORD=str, DATABASE_PORT=int, DATABASE_HOST=str, - # S3 (NOTE: Not used anywhere) - USE_AWS_FOR_MEDIA=(bool, False), - S3_AWS_ACCESS_KEY_ID=str, - S3_AWS_SECRET_ACCESS_KEY=str, - S3_STORAGE_BUCKET_NAME=str, - S3_REGION_NAME=str, + # Storage + # Static, Media configs + DJANGO_STATIC_URL=(str, "/static/"), + DJANGO_MEDIA_URL=(str, "/media/"), + # -- File System + DJANGO_STATIC_ROOT=(str, BASE_DIR / "storage/static"), # Where to store + DJANGO_MEDIA_ROOT=(str, BASE_DIR / "storage/media"), # Where to store + # -- S3 + USE_S3_BUCKET=(bool, False), + AWS_S3_ENDPOINT_URL=str, + AWS_S3_ACCESS_KEY_ID=str, + AWS_S3_SECRET_ACCESS_KEY=str, + AWS_S3_REGION=str, + AWS_S3_BUCKET_NAME=str, + # -- Azure blob storage + USE_AZURE_STORAGE=(bool, False), + AZURE_STORAGE_CONTAINER=str, + AZURE_STORAGE_CONNECTION_STRING=(str, None), + AZURE_STORAGE_ACCOUNT_NAME=str, + AZURE_STORAGE_ACCOUNT_KEY=(str, None), + AZURE_STORAGE_TOKEN_CREDENTIAL=(str, None), + AZURE_STORAGE_MANAGED_IDENTITY=(bool, False), # Redis CELERY_REDIS_URL=str, # redis://redis:6379/0 CACHE_REDIS_URL=str, # redis://redis:6379/1 @@ -66,9 +85,6 @@ DEBUG = env("DJANGO_DEBUG") -# Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve().parent.parent - ALLOWED_HOSTS = ["server", *env("DJANGO_ALLOWED_HOSTS")] @@ -246,32 +262,44 @@ def log_render_extra_context(record): # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.2/howto/static-files/ - - -if env("USE_AWS_FOR_MEDIA"): - AWS_S3_ACCESS_KEY_ID = env("S3_AWS_ACCESS_KEY_ID") - AWS_S3_SECRET_ACCESS_KEY = env("S3_AWS_SECRET_ACCESS_KEY") - AWS_STORAGE_BUCKET_NAME = env("S3_STORAGE_BUCKET_NAME") - AWS_S3_REGION_NAME = env("S3_REGION_NAME") +# https://docs.djangoproject.com/en/4.0/howto/static-files/ + +STATIC_URL = env("DJANGO_STATIC_URL") +MEDIA_URL = env("DJANGO_MEDIA_URL") + +if env("USE_AZURE_STORAGE"): + AZURE_CONNECTION_STRING = env("AZURE_STORAGE_CONNECTION_STRING") + if not AZURE_CONNECTION_STRING: + AZURE_ACCOUNT_NAME = env("AZURE_STORAGE_ACCOUNT_NAME") + AZURE_ACCOUNT_KEY = env("AZURE_STORAGE_ACCOUNT_KEY") + AZURE_TOKEN_CREDENTIAL = env("AZURE_STORAGE_TOKEN_CREDENTIAL") + if env("AZURE_STORAGE_MANAGED_IDENTITY"): + AZURE_TOKEN_CREDENTIAL = DefaultAzureCredential() + AZURE_CONTAINER = env("AZURE_STORAGE_CONTAINER") + + AZURE_OVERWRITE_FILES = False + DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" + STATICFILES_STORAGE = "storages.backends.azure_storage.AzureStorage" + +elif env("USE_S3_BUCKET"): + AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL") + AWS_S3_ACCESS_KEY_ID = env("AWS_S3_ACCESS_KEY_ID") + AWS_S3_SECRET_ACCESS_KEY = env("AWS_S3_SECRET_ACCESS_KEY") + AWS_S3_REGION_NAME = env("AWS_S3_REGION") + AWS_STORAGE_BUCKET_NAME = env("AWS_S3_BUCKET_NAME") AWS_S3_FILE_OVERWRITE = False - AWS_DEFAULT_ACL = "private" - DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" - -STATIC_ROOT = os.path.join(BASE_DIR, "storage/static") -STATIC_URL = "/static/" - -MEDIA_ROOT = os.path.join(BASE_DIR, "storage/media") -MEDIA_URL = "/media/" + STATICFILES_STORAGE = "storages.backends.s3boto3.S3StaticStorage" +else: + STATIC_ROOT = env("DJANGO_STATIC_ROOT") + MEDIA_ROOT = env("DJANGO_MEDIA_ROOT") # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" - # https://docs.celeryq.dev/en/stable/userguide/configuration.html CELERY_REDIS_URL = env("CELERY_REDIS_URL") CELERY_BROKER_URL = CELERY_REDIS_URL diff --git a/risk_module/tests.py b/risk_module/tests.py new file mode 100644 index 0000000..daa7b62 --- /dev/null +++ b/risk_module/tests.py @@ -0,0 +1,11 @@ +from django.test import TestCase + + +class FakeTest(TestCase): + """ + This test is for running migrations only + docker compose run --rm server ./manage.py test --keepdb -v 2 risk_module.tests.FakeTest + """ + + def test_fake(self): + pass diff --git a/scripts/run_prod.sh b/scripts/run_prod.sh deleted file mode 100755 index cce1ec6..0000000 --- a/scripts/run_prod.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -x - -python manage.py collectstatic --noinput & -python manage.py migrate --noinput - -gunicorn risk_module.wsgi:application --bind 0.0.0.0:80 diff --git a/scripts/run_worker.sh b/scripts/run_worker.sh deleted file mode 100755 index fa51c3e..0000000 --- a/scripts/run_worker.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -x - -# Using --max-tasks-per-child to handle memory leaks -celery -A risk_module worker --loglevel=info --max-tasks-per-child=20 diff --git a/scripts/run_worker_beat.sh b/scripts/run_worker_beat.sh deleted file mode 100755 index 294f77b..0000000 --- a/scripts/run_worker_beat.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -x - -celery -A risk_module beat --max-interval 3600 -l debug --scheduler django_celery_beat.schedulers:DatabaseScheduler diff --git a/uv.lock b/uv.lock index e956131..5cd8d31 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.8.1, <3.9" resolution-markers = [ "platform_machine != 'aarch64' and platform_machine != 'arm64'", @@ -67,6 +67,73 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, ] +[[package]] +name = "azure-common" +version = "1.1.28" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/71/f6f71a276e2e69264a97ad39ef850dca0a04fce67b12570730cb38d0ccac/azure-common-1.1.28.zip", hash = "sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3", size = 20914, upload-time = "2022-02-03T19:39:44.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/55/7f118b9c1b23ec15ca05d15a578d8207aa1706bc6f7c87218efffbbf875d/azure_common-1.1.28-py2.py3-none-any.whl", hash = "sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad", size = 14462, upload-time = "2022-02-03T19:39:42.417Z" }, +] + +[[package]] +name = "azure-core" +version = "1.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "six" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/aa/7c9db8edd626f1a7d99d09ef7926f6f4fb34d5f9fa00dc394afdfe8e2a80/azure_core-1.33.0.tar.gz", hash = "sha256:f367aa07b5e3005fec2c1e184b882b0b039910733907d001c20fb08ebb8c0eb9", size = 295633, upload-time = "2025-04-03T23:51:02.058Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/b7/76b7e144aa53bd206bf1ce34fa75350472c3f69bf30e5c8c18bc9881035d/azure_core-1.33.0-py3-none-any.whl", hash = "sha256:9b5b6d0223a1d38c37500e6971118c1e0f13f54951e6893968b38910bc9cda8f", size = 207071, upload-time = "2025-04-03T23:51:03.806Z" }, +] + +[[package]] +name = "azure-identity" +version = "1.21.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography" }, + { name = "msal" }, + { name = "msal-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/a1/f1a683672e7a88ea0e3119f57b6c7843ed52650fdcac8bfa66ed84e86e40/azure_identity-1.21.0.tar.gz", hash = "sha256:ea22ce6e6b0f429bc1b8d9212d5b9f9877bd4c82f1724bfa910760612c07a9a6", size = 266445, upload-time = "2025-03-11T20:53:07.463Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/9f/1f9f3ef4f49729ee207a712a5971a9ca747f2ca47d9cbf13cf6953e3478a/azure_identity-1.21.0-py3-none-any.whl", hash = "sha256:258ea6325537352440f71b35c3dffe9d240eae4a5126c1b7ce5efd5766bd9fd9", size = 189190, upload-time = "2025-03-11T20:53:09.197Z" }, +] + +[[package]] +name = "azure-storage-blob" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-common" }, + { name = "azure-storage-common" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/76/bd6eadc0f2b481bcfb43c37caacf77372401fc013c1431861561b794e06a/azure-storage-blob-2.1.0.tar.gz", hash = "sha256:b90323aad60f207f9f90a0c4cf94c10acc313c20b39403398dfba51f25f7b454", size = 83156, upload-time = "2019-08-02T04:24:19.926Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/84/610f379b46d7d3c2d48eadeed6a12b6d46a43100fea70534f5992d0ac996/azure_storage_blob-2.1.0-py2.py3-none-any.whl", hash = "sha256:a8e91a51d4f62d11127c7fd8ba0077385c5b11022f0269f8a2a71b9fc36bef31", size = 88133, upload-time = "2019-08-02T04:24:12.482Z" }, +] + +[[package]] +name = "azure-storage-common" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-common" }, + { name = "cryptography" }, + { name = "python-dateutil" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/12/e074fe454bc327fbe2a61e20d3260473ee4a0fd85387baf249dc83c8e774/azure-storage-common-2.1.0.tar.gz", hash = "sha256:ccedef5c67227bc4d6670ffd37cec18fb529a1b7c3a5e53e4096eb0cf23dc73f", size = 41869, upload-time = "2019-08-02T04:24:21.763Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/a0/6794b318ce0118d1a4053bdf0149a60807407db9b710354f2b203c2f5975/azure_storage_common-2.1.0-py2.py3-none-any.whl", hash = "sha256:b01a491a18839b9d05a4fe3421458a0ddb5ab9443c14e487f40d16f9a1dc2fbe", size = 47778, upload-time = "2019-08-02T04:24:14.67Z" }, +] + [[package]] name = "backcall" version = "0.2.0" @@ -177,6 +244,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/52/34c6cf5bb9285074dc3531c437b3919e825d976fde097a7a73f79e726d03/certifi-2025.7.14-py3-none-any.whl", hash = "sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2", size = 162722, upload-time = "2025-07-14T03:29:26.863Z" }, ] +[[package]] +name = "cffi" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/08/15bf6b43ae9bd06f6b00ad8a91f5a8fe1069d4c9fab550a866755402724e/cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b", size = 182457, upload-time = "2024-09-04T20:44:47.892Z" }, + { url = "https://files.pythonhosted.org/packages/c2/5b/f1523dd545f92f7df468e5f653ffa4df30ac222f3c884e51e139878f1cb5/cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964", size = 425932, upload-time = "2024-09-04T20:44:49.491Z" }, + { url = "https://files.pythonhosted.org/packages/53/93/7e547ab4105969cc8c93b38a667b82a835dd2cc78f3a7dad6130cfd41e1d/cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9", size = 448585, upload-time = "2024-09-04T20:44:51.671Z" }, + { url = "https://files.pythonhosted.org/packages/56/c4/a308f2c332006206bb511de219efeff090e9d63529ba0a77aae72e82248b/cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc", size = 456268, upload-time = "2024-09-04T20:44:53.51Z" }, + { url = "https://files.pythonhosted.org/packages/ca/5b/b63681518265f2f4060d2b60755c1c77ec89e5e045fc3773b72735ddaad5/cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c", size = 436592, upload-time = "2024-09-04T20:44:55.085Z" }, + { url = "https://files.pythonhosted.org/packages/bb/19/b51af9f4a4faa4a8ac5a0e5d5c2522dcd9703d07fac69da34a36c4d960d3/cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1", size = 446512, upload-time = "2024-09-04T20:44:57.135Z" }, + { url = "https://files.pythonhosted.org/packages/e2/63/2bed8323890cb613bbecda807688a31ed11a7fe7afe31f8faaae0206a9a3/cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8", size = 171576, upload-time = "2024-09-04T20:44:58.535Z" }, + { url = "https://files.pythonhosted.org/packages/2f/70/80c33b044ebc79527447fd4fbc5455d514c3bb840dede4455de97da39b4d/cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1", size = 181229, upload-time = "2024-09-04T20:44:59.963Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.2" @@ -254,6 +340,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "cryptography" +version = "46.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/04/ee2a9e8542e4fa2773b81771ff8349ff19cdd56b7258a0cc442639052edb/cryptography-46.0.5.tar.gz", hash = "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", size = 750064, upload-time = "2026-02-10T19:18:38.255Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/81/b0bb27f2ba931a65409c6b8a8b358a7f03c0e46eceacddff55f7c84b1f3b/cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", size = 7176289, upload-time = "2026-02-10T19:17:08.274Z" }, + { url = "https://files.pythonhosted.org/packages/ff/9e/6b4397a3e3d15123de3b1806ef342522393d50736c13b20ec4c9ea6693a6/cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", size = 4275637, upload-time = "2026-02-10T19:17:10.53Z" }, + { url = "https://files.pythonhosted.org/packages/63/e7/471ab61099a3920b0c77852ea3f0ea611c9702f651600397ac567848b897/cryptography-46.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", size = 4424742, upload-time = "2026-02-10T19:17:12.388Z" }, + { url = "https://files.pythonhosted.org/packages/37/53/a18500f270342d66bf7e4d9f091114e31e5ee9e7375a5aba2e85a91e0044/cryptography-46.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", size = 4277528, upload-time = "2026-02-10T19:17:13.853Z" }, + { url = "https://files.pythonhosted.org/packages/22/29/c2e812ebc38c57b40e7c583895e73c8c5adb4d1e4a0cc4c5a4fdab2b1acc/cryptography-46.0.5-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", size = 4947993, upload-time = "2026-02-10T19:17:15.618Z" }, + { url = "https://files.pythonhosted.org/packages/6b/e7/237155ae19a9023de7e30ec64e5d99a9431a567407ac21170a046d22a5a3/cryptography-46.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", size = 4456855, upload-time = "2026-02-10T19:17:17.221Z" }, + { url = "https://files.pythonhosted.org/packages/2d/87/fc628a7ad85b81206738abbd213b07702bcbdada1dd43f72236ef3cffbb5/cryptography-46.0.5-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", size = 3984635, upload-time = "2026-02-10T19:17:18.792Z" }, + { url = "https://files.pythonhosted.org/packages/84/29/65b55622bde135aedf4565dc509d99b560ee4095e56989e815f8fd2aa910/cryptography-46.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", size = 4277038, upload-time = "2026-02-10T19:17:20.256Z" }, + { url = "https://files.pythonhosted.org/packages/bc/36/45e76c68d7311432741faf1fbf7fac8a196a0a735ca21f504c75d37e2558/cryptography-46.0.5-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", size = 4912181, upload-time = "2026-02-10T19:17:21.825Z" }, + { url = "https://files.pythonhosted.org/packages/6d/1a/c1ba8fead184d6e3d5afcf03d569acac5ad063f3ac9fb7258af158f7e378/cryptography-46.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", size = 4456482, upload-time = "2026-02-10T19:17:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e5/3fb22e37f66827ced3b902cf895e6a6bc1d095b5b26be26bd13c441fdf19/cryptography-46.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", size = 4405497, upload-time = "2026-02-10T19:17:26.66Z" }, + { url = "https://files.pythonhosted.org/packages/1a/df/9d58bb32b1121a8a2f27383fabae4d63080c7ca60b9b5c88be742be04ee7/cryptography-46.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", size = 4667819, upload-time = "2026-02-10T19:17:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ed/325d2a490c5e94038cdb0117da9397ece1f11201f425c4e9c57fe5b9f08b/cryptography-46.0.5-cp311-abi3-win32.whl", hash = "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", size = 3028230, upload-time = "2026-02-10T19:17:30.518Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5a/ac0f49e48063ab4255d9e3b79f5def51697fce1a95ea1370f03dc9db76f6/cryptography-46.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", size = 3480909, upload-time = "2026-02-10T19:17:32.083Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fa/a66aa722105ad6a458bebd64086ca2b72cdd361fed31763d20390f6f1389/cryptography-46.0.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", size = 7170514, upload-time = "2026-02-10T19:17:56.267Z" }, + { url = "https://files.pythonhosted.org/packages/0f/04/c85bdeab78c8bc77b701bf0d9bdcf514c044e18a46dcff330df5448631b0/cryptography-46.0.5-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", size = 4275349, upload-time = "2026-02-10T19:17:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/32/9b87132a2f91ee7f5223b091dc963055503e9b442c98fc0b8a5ca765fab0/cryptography-46.0.5-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", size = 4420667, upload-time = "2026-02-10T19:18:00.619Z" }, + { url = "https://files.pythonhosted.org/packages/a1/a6/a7cb7010bec4b7c5692ca6f024150371b295ee1c108bdc1c400e4c44562b/cryptography-46.0.5-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", size = 4276980, upload-time = "2026-02-10T19:18:02.379Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7c/c4f45e0eeff9b91e3f12dbd0e165fcf2a38847288fcfd889deea99fb7b6d/cryptography-46.0.5-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", size = 4939143, upload-time = "2026-02-10T19:18:03.964Z" }, + { url = "https://files.pythonhosted.org/packages/37/19/e1b8f964a834eddb44fa1b9a9976f4e414cbb7aa62809b6760c8803d22d1/cryptography-46.0.5-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", size = 4453674, upload-time = "2026-02-10T19:18:05.588Z" }, + { url = "https://files.pythonhosted.org/packages/db/ed/db15d3956f65264ca204625597c410d420e26530c4e2943e05a0d2f24d51/cryptography-46.0.5-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", size = 3978801, upload-time = "2026-02-10T19:18:07.167Z" }, + { url = "https://files.pythonhosted.org/packages/41/e2/df40a31d82df0a70a0daf69791f91dbb70e47644c58581d654879b382d11/cryptography-46.0.5-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", size = 4276755, upload-time = "2026-02-10T19:18:09.813Z" }, + { url = "https://files.pythonhosted.org/packages/33/45/726809d1176959f4a896b86907b98ff4391a8aa29c0aaaf9450a8a10630e/cryptography-46.0.5-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", size = 4901539, upload-time = "2026-02-10T19:18:11.263Z" }, + { url = "https://files.pythonhosted.org/packages/99/0f/a3076874e9c88ecb2ecc31382f6e7c21b428ede6f55aafa1aa272613e3cd/cryptography-46.0.5-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", size = 4452794, upload-time = "2026-02-10T19:18:12.914Z" }, + { url = "https://files.pythonhosted.org/packages/02/ef/ffeb542d3683d24194a38f66ca17c0a4b8bf10631feef44a7ef64e631b1a/cryptography-46.0.5-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", size = 4404160, upload-time = "2026-02-10T19:18:14.375Z" }, + { url = "https://files.pythonhosted.org/packages/96/93/682d2b43c1d5f1406ed048f377c0fc9fc8f7b0447a478d5c65ab3d3a66eb/cryptography-46.0.5-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", size = 4667123, upload-time = "2026-02-10T19:18:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/45/2d/9c5f2926cb5300a8eefc3f4f0b3f3df39db7f7ce40c8365444c49363cbda/cryptography-46.0.5-cp38-abi3-win32.whl", hash = "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", size = 3010220, upload-time = "2026-02-10T19:18:17.361Z" }, + { url = "https://files.pythonhosted.org/packages/48/ef/0c2f4a8e31018a986949d34a01115dd057bf536905dca38897bacd21fac3/cryptography-46.0.5-cp38-abi3-win_amd64.whl", hash = "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", size = 3467050, upload-time = "2026-02-10T19:18:18.899Z" }, +] + [[package]] name = "decorator" version = "5.2.1" @@ -383,6 +509,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/3d/9cf5b269de293098c1033ec27bc63e58f06164ca6deeda58d540536096a4/django_storages-1.11.1-py3-none-any.whl", hash = "sha256:f28765826d507a0309cfaa849bd084894bc71d81bf0d09479168d44785396f80", size = 42834, upload-time = "2020-12-24T00:37:37.089Z" }, ] +[package.optional-dependencies] +azure = [ + { name = "azure-storage-blob" }, +] + [[package]] name = "django-stubs" version = "4.2.6" @@ -767,6 +898,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899, upload-time = "2024-04-15T13:44:43.265Z" }, ] +[[package]] +name = "msal" +version = "1.35.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/aa/5a646093ac218e4a329391d5a31e5092a89db7d2ef1637a90b82cd0b6f94/msal-1.35.1.tar.gz", hash = "sha256:70cac18ab80a053bff86219ba64cfe3da1f307c74b009e2da57ef040eb1b5656", size = 165658, upload-time = "2026-03-04T23:38:51.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/86/16815fddf056ca998853c6dc525397edf0b43559bb4073a80d2bc7fe8009/msal-1.35.1-py3-none-any.whl", hash = "sha256:8f4e82f34b10c19e326ec69f44dc6b30171f2f7098f3720ea8a9f0c11832caa3", size = 119909, upload-time = "2026-03-04T23:38:50.452Z" }, +] + +[[package]] +name = "msal-extensions" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/a3/0b9bd5d32aaa40b1c989c4fd51406ccf491510bfad141f5e595c46f33c04/msal_extensions-1.3.0.tar.gz", hash = "sha256:96918996642b38c78cd59b55efa0f06fd1373c90e0949be8615697c048fba62c", size = 24406, upload-time = "2025-03-13T23:02:40.466Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/3f/cea513102479bb84c2d6eaad23dc05a85933d5da88c7a5f16f37116ad39a/msal_extensions-1.3.0-py3-none-any.whl", hash = "sha256:105328ddcbdd342016c9949d8f89e3917554740c8ab26669c0fa0e069e730a0e", size = 28611, upload-time = "2025-03-13T23:02:39.545Z" }, +] + [[package]] name = "mypy" version = "1.14.1" @@ -963,6 +1120,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "pycparser" +version = "2.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cf/d2d3b9f5699fb1e4615c8e32ff220203e43b248e1dfcc6736ad9057731ca/pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", size = 173734, upload-time = "2025-09-09T13:23:47.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934", size = 118140, upload-time = "2025-09-09T13:23:46.651Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -972,6 +1138,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pyjwt" +version = "2.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/68/ce067f09fca4abeca8771fe667d89cc347d1e99da3e093112ac329c6020e/pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c", size = 78825, upload-time = "2024-08-01T15:01:08.445Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/84/0fdf9b18ba31d69877bd39c9cd6052b47f3761e9910c15de788e519f079f/PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850", size = 22344, upload-time = "2024-08-01T15:01:06.481Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pyproj" version = "3.3.1" @@ -1152,6 +1332,7 @@ name = "risk-module-server" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "azure-identity" }, { name = "beautifulsoup4" }, { name = "boto3" }, { name = "celery", extra = ["redis"] }, @@ -1164,7 +1345,7 @@ dependencies = [ { name = "django-filter" }, { name = "django-health-check" }, { name = "django-redis" }, - { name = "django-storages" }, + { name = "django-storages", extra = ["azure"] }, { name = "djangorestframework" }, { name = "djangorestframework-camel-case" }, { name = "drf-spectacular" }, @@ -1196,6 +1377,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "azure-identity" }, { name = "beautifulsoup4", specifier = "==4.11.1" }, { name = "boto3", specifier = "==1.20.33" }, { name = "celery", extras = ["redis"], specifier = "==5.1.1" }, @@ -1208,7 +1390,7 @@ requires-dist = [ { name = "django-filter", specifier = "==2.4.0" }, { name = "django-health-check" }, { name = "django-redis", specifier = "==5.0.0" }, - { name = "django-storages", specifier = "==1.11.1" }, + { name = "django-storages", extras = ["azure"], specifier = "~=1.11.1" }, { name = "djangorestframework", specifier = "==3.12.4" }, { name = "djangorestframework-camel-case", specifier = "==1.2.0" }, { name = "drf-spectacular" },