.gitlab-ci.yml

# Please see `docs/development/ci-cd.md` for more information.

include: "/.gitlab/ci_includes/jobs.yml"

stages:
  - build
  - lint
  - test
  - build-final
  - publish

variables:
  # Visit https://gitlab.com/baserow/baserow/-/pipelines/new select your branch
  # and click run to be able to run a new pipeline where you can manually control these
  # variables.
  TRIGGER_FULL_IMAGE_REBUILD:
    value: "no"
    description: "If set to yes then all images will re-pull their base images and rebuild entirely from scratch with no caching."
  ENABLE_JOB_SKIPPING:
    value: "false"
    description: "If set to true then tests and lints will skipped when safe to do so."
  ENABLE_COVERAGE:
    value: "true"
    description: "If set to false then tests will not generate coverage or testing reports used by gitlab to show nicer MRs."
  ENABLE_RELEASES:
    value: "false"
    description: "If true then on master and develop release images will be pushed automatically"
  BUILD_ALL_IN_ONE:
    value: "false"
    description: "If true then regardless of what branch we are on the all in one image will be built."
  # An image repo which is used for storing and passing images between ci pipeline jobs
  # and also speeding up ci builds by caching from the latest ci image when building.
  CI_IMAGE_REPO: $CI_REGISTRY_IMAGE/ci
  # Any images with tags prefixed with the two variables below will be cleaned up automatically
  # by our gitlab cleanup job:
  # (https://gitlab.com/baserow/baserow/-/settings/packages_and_registries).
  #
  # ## Note:
  #   These cleanup tag prefixes are needed as gitlab only supports cleanup by defining
  #   a regex that matches tags, so we can't do cleanup differently based on image name
  #   or repo...
  #
  # IMPORTANT: UPDATE GITLAB CONTAINER REPO CLEANUP JOB REGEX IF YOU CHANGE THIS
  CLEANUP_JOB_CI_TAG_PREFIX: ci-latest-
  # IMPORTANT: UPDATE GITLAB CONTAINER REPO CLEANUP JOB REGEX IF YOU CHANGE THIS
  TESTED_IMAGE_PREFIX: ci-tested-
  BACKEND_IMAGE_NAME: backend
  BACKEND_DEV_IMAGE_NAME: backend_dev
  WEBFRONTEND_IMAGE_NAME: web-frontend
  ALLINONE_IMAGE_NAME: baserow
  CLOUDRON_IMAGE_NAME: cloudron
  HEROKU_IMAGE_NAME: heroku
  WEBFRONTEND_DEV_IMAGE_NAME: web-frontend_dev
  BACKEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$BACKEND_DEV_IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$CI_COMMIT_SHORT_SHA
  WEBFRONTEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$WEBFRONTEND_DEV_IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$CI_COMMIT_SHORT_SHA
  # Once images are tested they will publish under these names to ensure that any
  # tag only runs of the pipeline can never publish untested images.
  TESTED_BACKEND_CI_IMAGE: $CI_IMAGE_REPO/$BACKEND_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_WEBFRONTEND_CI_IMAGE: $CI_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_BACKEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$BACKEND_DEV_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_WEBFRONTEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$WEBFRONTEND_DEV_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_ALLINONE_CI_IMAGE: $CI_IMAGE_REPO/$ALLINONE_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_CLOUDRON_CI_IMAGE: $CI_IMAGE_REPO/$CLOUDRON_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_HEROKU_CI_IMAGE: $CI_IMAGE_REPO/$HEROKU_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  # Used to tag the latest images on $DEVELOP_BRANCH_NAME
  DEVELOP_LATEST_TAG: develop-latest
  # Names of important branches used to decide when to run certain jobs.
  MASTER_BRANCH_NAME: master
  DEVELOP_BRANCH_NAME: develop
  # The locations of the various dockerfiles to build.
  BACKEND_DOCKERFILE_PATH: $CI_PROJECT_DIR/backend/Dockerfile
  WEBFRONTEND_DOCKERFILE_PATH: $CI_PROJECT_DIR/web-frontend/Dockerfile
  ALLINONE_DOCKERFILE_PATH: $CI_PROJECT_DIR/deploy/all-in-one/Dockerfile
  CLOUDRON_DOCKERFILE_PATH: $CI_PROJECT_DIR/deploy/cloudron/Dockerfile
  HEROKU_DOCKERFILE_PATH: $CI_PROJECT_DIR/heroku.Dockerfile
  # By default, forks can use the baserow projects ci util image so they don't have
  # to run the manual job to build their own. If you want to build and use your own
  # then set PROJECT_PATH_FOR_CI_UTIL_IMAGE in your gitlab ci settings as a project
  # level variable to whichever project you want.
  PROJECT_PATH_FOR_CI_UTIL_IMAGE: $CI_PROJECT_PATH
  PROJECT_PATH_FOR_E2E_TEST_IMAGE: $CI_PROJECT_PATH
  PROJECT_PATH_FOR_CI_DIND_IMAGE: $CI_PROJECT_PATH
  # The image path for the helper CI util image that will be built and pushed to.
  CI_UTIL_IMAGE: $CI_REGISTRY/$PROJECT_PATH_FOR_CI_UTIL_IMAGE/ci/ci_util_image:latest
  # The image path for the dind CI image that will be built and pushed to.
  CI_DIND_IMAGE: $CI_REGISTRY/$PROJECT_PATH_FOR_E2E_TEST_IMAGE/ci/ci_dind_image:latest

# ==================================== CI UTIL ====================================

# A simple util image used by the other jobs containing some helper tools like git, jq,
# coverage etc.
build-ci-util-image:
  image: docker:20.10.18
  stage: build
  services:
    - name: $CI_DIND_IMAGE
      alias: docker
  variables:
    DOCKER_BUILDKIT: 1
    DOCKER_HOST: tcp://docker:2376
    DOCKER_TLS_CERTDIR: "/certs"
  script:
    - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
    - cd .gitlab/ci_util_image
    - docker build -t $CI_UTIL_IMAGE .
    - docker push $CI_UTIL_IMAGE
  # Only trigger this job manually to prevent it running every single time a new branch
  # is made. See https://gitlab.com/gitlab-org/gitlab/-/issues/11427
  when: manual
  # We can't use the newer rules: syntax due to https://gitlab.com/gitlab-org/gitlab/-/issues/34756
  only:
    changes:
      - .gitlab/ci_util_image/*
      - .gitlab-ci.yml
  except:
    refs:
      # When a pipeline is triggered by an upstream project we don't want to rebuild.
      - pipelines
      # When a pipeline is triggered by a git commit tag we don't want to rebuild.
      - tags

# An image used by the e2e tests.
build-ci-dind-image:
  image: docker:20.10.18
  stage: build
  services:
    - docker:20.10.18-dind
  variables:
    DOCKER_BUILDKIT: 1
    DOCKER_HOST: tcp://docker:2376
    DOCKER_TLS_CERTDIR: "/certs"
  script:
    - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
    - cd .gitlab/ci_dind_image
    - docker build -t $CI_DIND_IMAGE .
    - docker push $CI_DIND_IMAGE
  # Only trigger this job manually to prevent it running every single time a new branch
  # is made. See https://gitlab.com/gitlab-org/gitlab/-/issues/11427
  when: manual
  # We can't use the newer rules: syntax due to https://gitlab.com/gitlab-org/gitlab/-/issues/34756
  only:
    changes:
      - .gitlab/ci_dind_image/*
      - .gitlab-ci.yml
  except:
    refs:
      # When a pipeline is triggered by an upstream project we don't want to rebuild.
      - pipelines
      # When a pipeline is triggered by a git commit tag we don't want to rebuild.
      - tags
# ==================================== BACKEND ====================================

# If pipeline not triggered by tag :
# - Builds the backend dev image and stores in ci repo for next stages.
build-backend-image:
  extends: .build-baserow-image
  variables:
    DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $BACKEND_DOCKERFILE_PATH

# If pipeline not triggered by tag:
# - Runs the backend lint if changes to the backend, otherwise skips.
backend-lint:
  stage: lint
  extends:
    - .docker-image-test-stage
    - .skippable-job
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/ enterprise/backend/"
  script:
    - docker run --rm $BACKEND_CI_DEV_IMAGE lint
  needs:
    - job: build-backend-image

backend-check-startup-python:
  extends:
    - .docker-image-test-stage
    - .skippable-job
    - .requires-lint
  services:
    - name: $CI_DIND_IMAGE
      alias: docker
    - name: postgres:13
      alias: db
  variables:
    DOCKER_HOST: tcp://docker:2376
    DOCKER_TLS_CERTDIR: "/certs"
    POSTGRES_USER: baserow
    POSTGRES_PASSWORD: baserow
    POSTGRES_DB: baserow
    RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/ enterprise/backend/ .gitlab-ci.yml"
    FF_NETWORK_PER_BUILD: 1
  script:
    - docker run --rm --network=host $BACKEND_CI_DEV_IMAGE ci-check-startup
    - docker run --rm --network=host $BACKEND_CI_DEV_IMAGE ci-check-startup-oss-only

# If pipeline not triggered by tag:
# - Runs the backend tests (the first 1/3) if changes to the backend, otherwise skips.
# - Generates coverage db's and stores as artifact for later coverage merge and report
backend-test-group-1:
  extends:
    - .docker-image-test-stage
    - .skippable-job
    - .requires-lint
  services:
    - name: $CI_DIND_IMAGE
      alias: docker
    - name: postgres:13
      alias: db
      command:
        [
          "postgres",
          "-c",
          "fsync=off",
          "-c",
          "synchronous_commit=off",
          "-c",
          "full_page_writes=off",
        ]
  variables:
    DOCKER_HOST: tcp://docker:2376
    DOCKER_TLS_CERTDIR: "/certs"
    POSTGRES_USER: baserow
    POSTGRES_PASSWORD: baserow
    POSTGRES_DB: baserow
    PYTEST_SPLIT_GROUP: 1
    RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/ enterprise/backend/ .gitlab-ci.yml"
    DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: "true"
    FF_NETWORK_PER_BUILD: 1
  script:
    - mkdir -p reports
    - TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")
    - PYTEST_EXTRA_ARGS=$([[ "$RUN_MORNING_TESTS" = "true" ]] && echo '--run-once-per-day-in-ci' || echo "")
    - |
      docker run \
        -e PYTEST_SPLITS=10 \
        -e PYTEST_SPLIT_GROUP=$PYTEST_SPLIT_GROUP \
        --name=baserow_backend_test_container \
        --network=host \
        $BACKEND_CI_DEV_IMAGE $TEST_TYPE $PYTEST_EXTRA_ARGS;
    - docker cp baserow_backend_test_container:/baserow/backend/reports .
    - docker rm baserow_backend_test_container

  artifacts:
    name: "$CI_JOB_NAME-reports"
    paths:
      - reports/
    reports:
      junit: reports/report.xml

# Create 2 more separate groups to parallelize pytest by using separate groups to
# decrease overall build time. Pytest xdist doesn't help as the gitlab saas runners only
# have a single virtual core so `pytest -n 2+` slows things down.
backend-test-group-2:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 2

backend-test-group-3:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 3

backend-test-group-4:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 4

backend-test-group-5:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 5

backend-test-group-6:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 6

backend-test-group-7:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 7

backend-test-group-8:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 8

backend-test-group-9:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 9

backend-test-group-10:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 10

# Collects together all the separate backend coverage databases from previous jobs and
# combines them to generate a single report for gitlab to use. GitLab itself does not
# correctly merge these if you just add them all separately into artifacts->reports->
# cobertura.
collect-backend-coverage:
  image: $CI_UTIL_IMAGE
  stage: build-final
  interruptible: true
  only:
    variables:
      - $ENABLE_COVERAGE == "true"
  # Prevent rebuilds when tagging as all we want to do is tag and push
  except:
    refs:
      - tags
  # Depend on the `reports` artifacts from the previous jobs
  needs:
    - job: backend-test-group-1
      artifacts: true
    - job: backend-test-group-2
      artifacts: true
    - job: backend-test-group-3
      artifacts: true
    - job: backend-test-group-4
      artifacts: true
    - job: backend-test-group-5
      artifacts: true
    - job: backend-test-group-6
      artifacts: true
    - job: backend-test-group-7
      artifacts: true
    - job: backend-test-group-8
      artifacts: true
    - job: backend-test-group-9
      artifacts: true
    - job: backend-test-group-10
      artifacts: true
  script:
    - . /baserow/venv/bin/activate
    # The reports artifacts will be extracted before the script runs into reports by
    # gitlab
    - cp reports/.coverage.* .
    - export COVERAGE_RCFILE=backend/.coveragerc
    - coverage combine
    - coverage report
    - coverage xml -o coverage.xml
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
  coverage: '/^TOTAL.+?(\d+\%)$/'

# If pipeline not triggered by tag:
# - Build and store non-dev images in CI repo under the `ci-tested` tag so we know
#   those images have passed the tests.
build-final-backend-image:
  extends: .build-final-baserow-image
  needs:
    - job: backend-test-group-1
    - job: backend-test-group-2
    - job: backend-test-group-3
    - job: backend-test-group-4
    - job: backend-test-group-5
    - job: backend-test-group-6
    - job: backend-test-group-7
    - job: backend-test-group-8
    - job: backend-test-group-9
    - job: backend-lint
  variables:
    IMAGE_NAME: $BACKEND_IMAGE_NAME
    DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $BACKEND_DOCKERFILE_PATH

build-final-backend-image-manual:
  extends: .build-final-baserow-image
  needs:
    - job: build-backend-image
  except:
    variables:
      - $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
      - $CI_COMMIT_MESSAGE =~ /\[build-all]/
  only:
  when: manual
  variables:
    IMAGE_NAME: $BACKEND_IMAGE_NAME
    DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $BACKEND_DOCKERFILE_PATH

# ==================================== WEB-FRONTEND ====================================

# If pipeline not triggered by tag:
# - Builds the web-frontend dev image and stores in ci repo for next stages.
build-web-frontend-image:
  extends: .build-baserow-image
  variables:
    DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH

# If pipeline not triggered by tag:
# - Runs eslint and stylelint if the web-frontend code has changed, otherwise skips.
web-frontend-lint:
  stage: lint
  extends:
    - .docker-image-test-stage
    - .skippable-job
  needs:
    - job: build-web-frontend-image
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/ enterprise/web-frontend/"
  script:
    - docker run --rm $WEBFRONTEND_CI_DEV_IMAGE lint

# If pipeline not triggered by tag:
# - Runs the web-frontend tests if the web-frontend has changed, otherwise skips.
# - Generates coverage and testing reports
web-frontend-test:
  extends:
    - .docker-image-test-stage
    - .skippable-job
    - .requires-lint
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/ enterprise/web-frontend/"
    DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: "true"
  script:
    - mkdir reports/ -p
    - TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")
    - |
      docker run --name=webfrontend_test $WEBFRONTEND_CI_DEV_IMAGE $TEST_TYPE \
        | tee reports/stdout.txt;
    - docker cp webfrontend_test:/baserow/reports .
    - docker rm webfrontend_test
  artifacts:
    paths:
      - reports/
    reports:
      junit: reports/junit.xml
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
  coverage: '/Lines\s*:\s*(\d+.?\d*)%/'

zapier-integration-test:
  extends:
    - .docker-image-test-stage
    - .skippable-job
    - .requires-lint
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "integrations/zapier"
  script:
    - cd integrations/zapier
    - yarn install
    - yarn run zapier test

# If pipeline not triggered by tag:
# - Build and store non-dev images in CI repo under the `ci-tested` tag so we know
#   those images have passed the tests.
build-final-web-frontend-image:
  extends:
    - .build-final-baserow-image
  variables:
    IMAGE_NAME: $WEBFRONTEND_IMAGE_NAME
    DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH
  needs:
    - job: web-frontend-test

# Allow to manually build the image from any branch
build-final-web-frontend-image-manual:
  extends: .build-final-baserow-image
  needs:
    - job: build-web-frontend-image
  except:
    variables:
      - $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
      - $CI_COMMIT_MESSAGE =~ /\[build-all]/
  only:
  when: manual
  variables:
    IMAGE_NAME: $WEBFRONTEND_IMAGE_NAME
    DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH

# ==================================== E2E TESTS ======================================

e2e-tests-group-1:
  image: mcr.microsoft.com/playwright:v1.48.2
  stage: test
  services:
    - name: postgres:13
      alias: db
      variables:
        POSTGRES_USER: baserow
        POSTGRES_PASSWORD: baserow
        POSTGRES_DB: baserow
    - name: adobe/s3mock:3.12.0
      alias: s3mock
    - name: redis:6-alpine
      alias: redis
    - name: $BACKEND_CI_DEV_IMAGE
      alias: backend
      command: ["gunicorn"]
    - name: $WEBFRONTEND_CI_DEV_IMAGE
      alias: web-frontend
      command: ["nuxt-dev-no-attach"]
    - name: $BACKEND_CI_DEV_IMAGE
      alias: celery
      command: ["celery-worker"]
      variables:
        BASEROW_RUN_MINIMAL: "yes"
  except:
    variables:
      - $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  variables:
    CI: 1
    SHARD_INDEX: 1
    FF_NETWORK_PER_BUILD: "true"
    initialBuckets: testbucket
    REDIS_URL: "redis://redis:6379"
    BASEROW_BACKEND_URL: "http://backend:8000"
    BASEROW_FRONTEND_URL: "http://frontend:3000"
    BASEROW_AMOUNT_OF_WORKERS: 1
    SECRET_KEY: test
    AWS_ACCESS_KEY_ID: anyvalue
    AWS_SECRET_ACCESS_KEY: anyvalue
    AWS_STORAGE_BUCKET_NAME: testbucket
    AWS_S3_ENDPOINT_URL: http://s3mock:9090
    AWS_S3_CUSTOM_DOMAIN: localhost:9090/testbucket
    AWS_S3_USE_SSL: no
    AWS_S3_URL_PROTOCOL: "http:"
    FEATURE_FLAGS: "*"
    PUBLIC_BACKEND_URL: http://backend:8000
    PUBLIC_WEB_FRONTEND_URL: http://web-frontend:3000
    NODE_OPTIONS: --max-old-space-size=8192
    BASEROW_TRIGGER_SYNC_TEMPLATES_AFTER_MIGRATION: false
    DJANGO_SETTINGS_MODULE: "baserow.config.settings.e2e"
    #DEBUG: "pw:api"
  script:
    - cd e2e-tests
    - ./wait-for-services.sh & yarn install & wait
    - npx playwright test --grep-invert=@slow --shard=$SHARD_INDEX/4 --project=firefox
  artifacts:
    name: "$CI_JOB_NAME-e2e-report"
    paths:
      # Save the HTML report as an artifact for download or visualization
      - e2e-tests/playwright-report
      - e2e-tests/blob-report/
    expire_in: 1 week
    when: always

e2e-tests-group-2:
  extends: e2e-tests-group-1
  variables:
    SHARD_INDEX: 2

e2e-tests-group-3:
  extends: e2e-tests-group-1
  variables:
    SHARD_INDEX: 3

e2e-tests-group-4:
  extends: e2e-tests-group-1
  variables:
    SHARD_INDEX: 4

collect-e2e-test-coverage:
  stage: build-final
  extends:
    - .docker-image-test-stage
  only:
    variables:
      - $ENABLE_COVERAGE == "true"
  # Prevent rebuilds when tagging as all we want to do is tag and push
  except:
    refs:
      - tags
    variables:
      - $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  interruptible: true
  needs:
    - job: e2e-tests-group-1
      artifacts: true
    - job: e2e-tests-group-2
      artifacts: true
    - job: e2e-tests-group-3
      artifacts: true
    - job: e2e-tests-group-4
      artifacts: true
  script:
    - cd e2e-tests
    - npx playwright merge-reports --reporter html blob-report/
  artifacts:
    name: "e2e-report"
    paths:
      - playwright-report/index.html
      - e2e-tests/playwright-report
    expire_in: 1 week
    when: always
    expose_as: "Playwright Report"

# ================================== ALL IN ONES ======================================
# If pipeline not triggered by tag:
# - Build and store the all-in-one image in CI repo under the `ci-tested` tag so we know
#   those images have passed the tests.
build-all-in-one-image:
  extends: .build-final-baserow-image
  needs:
    - job: build-final-web-frontend-image
    - job: build-final-backend-image
  tags:
    - saas-linux-medium-amd64
  only:
    variables:
      - $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
      - $BUILD_ALL_IN_ONE == "true"
      - $CI_COMMIT_MESSAGE =~ /\[build-all]/
  variables:
    IMAGE_NAME: $ALLINONE_IMAGE_NAME
    DOCKERFILE_PATH: $ALLINONE_DOCKERFILE_PATH
    BUILD_FROM_BACKEND_IMAGE: $TESTED_BACKEND_CI_IMAGE
    BUILD_FROM_WEBFRONTEND_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE

# If pipeline not triggered by tag:
# - Build and store cloudron image in CI repo under the `ci-tested` tag so we know
#   those images have passed the tests.
build-cloudron-image:
  extends: .build-final-baserow-image
  needs:
    - job: build-all-in-one-image
  tags:
    - saas-linux-medium-amd64
  only:
    variables:
      - $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME && $CLOUDRON_IMAGE_NAME != "disable"
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME && $CLOUDRON_IMAGE_NAME != "disable"
      # This rule and variable allows us to force build cloudron images on branches.
      - ($BUILD_ALL_IN_ONE == "true" || $CI_COMMIT_MESSAGE =~ /\[build-all]/ ) && $CLOUDRON_IMAGE_NAME != "disable"
  variables:
    IMAGE_NAME: $CLOUDRON_IMAGE_NAME
    DOCKERFILE_PATH: $CLOUDRON_DOCKERFILE_PATH
    BUILD_FROM_IMAGE: $TESTED_ALLINONE_CI_IMAGE

# We build the heroku image to simply test it builds and ensure we can have a smoke
# test in the future.
#
# If pipeline not triggered by tag:
# - Build and store heroku image in CI repo under the `ci-tested` tag so we know
#   those images have passed the tests.
build-heroku-image:
  extends: .build-final-baserow-image
  needs:
    - job: build-all-in-one-image
  tags:
    - saas-linux-medium-amd64
  only:
    variables:
      - ( $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME || $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME ) && ( $BUILD_ALL_IN_ONE == "true" || $CI_COMMIT_MESSAGE =~ /\[build-all]/ ) && $HEROKU_IMAGE_NAME != "disable"
  variables:
    IMAGE_NAME: $HEROKU_IMAGE_NAME
    DOCKERFILE_PATH: $HEROKU_DOCKERFILE_PATH
    BUILD_FROM_IMAGE: $TESTED_ALLINONE_CI_IMAGE

# ================================== TRIGGER SAAS =====================================

# Triggers a special pipeline in dependant project and passes various variables to it.
# Only on master and develop.
trigger-saas-build:
  stage: publish
  inherit:
    variables:
      - CI_COMMIT_BRANCH
      - TESTED_BACKEND_CI_IMAGE
      - TESTED_WEBFRONTEND_CI_IMAGE
      - CI_COMMIT_SHA
      - CI_COMMIT_SHORT_SHA
      - DEVELOP_BRANCH_NAME
      - MASTER_BRANCH_NAME
      - CI_IMAGE_REPO
      - BACKEND_IMAGE_NAME
      - WEBFRONTEND_IMAGE_NAME
      - TESTED_IMAGE_PREFIX
  variables:
    UPSTREAM_SHA: $CI_COMMIT_SHA
    UPSTREAM_SHORT_SHA: $CI_COMMIT_SHORT_SHA
  rules:
    - if: $CI_PROJECT_ID == "10858056" && $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  allow_failure: true
  trigger:
    project: baserow/baserow-saas
    branch: $CI_COMMIT_BRANCH

# ================================== PUSHING BACKEND ==================================

# Push baserow/backend:develop-latest
publish-backend-develop-latest-image:
  extends: .publish-baserow-image
  only:
    variables:
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$DEVELOP_LATEST_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/backend:$VERSION_GIT_TAG
publish-backend-release-tagged-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$CI_COMMIT_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/backend:latest
publish-backend-latest-release-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
    variables:
      - $CI_COMMIT_TAG =~ /^[0-9.]+$/
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:latest"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# ================================ PUSHING WEB-FRONTEND ===============================

# Push baserow/web-frontend:develop-latest
publish-webfrontend-develop-latest-image:
  extends: .publish-baserow-image
  only:
    variables:
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$DEVELOP_LATEST_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/web-frontend:$VERSION_GIT_TAG
publish-webfrontend-release-tagged-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$CI_COMMIT_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/web-frontend:latest
publish-webfrontend-latest-release-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
    variables:
      - $CI_COMMIT_TAG =~ /^[0-9.]+$/
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:latest"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# ================================ PUSHING ALL-IN-ONE ===============================

# Push baserow/baserow:develop-latest
publish-all-in-one-develop-latest-image:
  extends: .publish-baserow-image
  only:
    variables:
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_ALLINONE_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_IMAGE_NAME:$DEVELOP_LATEST_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/baserow:$VERSION_GIT_TAG
publish-all-in-one-release-tagged-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_ALLINONE_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_IMAGE_NAME:$CI_COMMIT_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/baserow:latest
publish-all-in-one-latest-release-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
    variables:
      - $CI_COMMIT_TAG =~ /^[0-9.]+$/
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_ALLINONE_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_IMAGE_NAME:latest"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/cloudron:develop-latest
publish-cloudron-develop-latest-image:
  extends: .publish-baserow-image
  only:
    variables:
      - $CLOUDRON_IMAGE_NAME != "disable" && $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_CLOUDRON_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$CLOUDRON_IMAGE_NAME:$DEVELOP_LATEST_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/cloudron:$VERSION_GIT_TAG
publish-cloudron-release-tagged-image:
  extends: .publish-baserow-image
  only:
    variables:
      - $CLOUDRON_IMAGE_NAME != "disable"
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_CLOUDRON_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$CLOUDRON_IMAGE_NAME:$CI_COMMIT_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# We don't push any heroku images as Heroku itself will build and use heroku.Dockerfile

# ================================ MISC LINTS ===============================

docker-file-hadolint:
  stage: lint
  extends:
    - .docker-image-test-stage
    - .skippable-job
  dependencies: []
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "Dockerfile"
  script:
    - mkdir -p reports
    # Ignore the version pinning as we want security upgrades ASAP in our docker images.
    - |
      docker run --rm -i -v "$(pwd)":/opt/hadolint/. -w /opt/hadolint \
        hadolint/hadolint:2.9.3-debian \
        hadolint --ignore DL3008 -f gitlab_codeclimate \
        backend/Dockerfile \
        web-frontend/Dockerfile \
        heroku.Dockerfile \
        deploy/*/Dockerfile > reports/hadolint.json
  artifacts:
    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    expire_in: 1 day
    when: always
    reports:
      codequality:
        - "reports/*"
    paths:
      - "reports/*"
  needs:
    - job: backend-lint

mjml-compiled-check:
  extends:
    - .docker-image-test-stage
    - .skippable-job
    - .requires-lint
  variables:
    RUN_WHEN_CHANGES_MADE_IN: ".eta"
  dependencies: []
  script:
    - cd backend/email_compiler
    - yarn install
    - yarn run compile
    - |
      git diff --exit-code || \
        (echo "Uncompiled changes found to mjml email templates, run the compiler in backend/email_compiler/ and committed the changes" && exit 1)

publish-helm-chart:
  image: dtzar/helm-kubectl:3.15.2
  stage: publish
  variables:
    HELM_REPO_PROJECT_ID: 59591209 # https://gitlab.com/baserow/baserow-chart
  only:
    refs:
      - tags
  artifacts:
    paths:
      - deploy/helm/baserow-*.tgz
  script:
    - cd deploy/helm
    - rm baserow/Chart.lock
    - helm repo add bitnami https://charts.bitnami.com/bitnami
    - helm repo add caddy https://caddyserver.github.io/ingress
    - helm dependency build baserow
    - helm package baserow
    - ls baserow-*.tgz
    - echo "Upload to chart repository"
    - CHART_FILE=$(ls baserow-*.tgz)
    - |
      HELM_CHART_VERSION=$(helm show chart baserow | grep version | tail -n 1 | awk -F ': ' '{print $2}')
    - |
      curl --request POST \
      --form token=$CI_JOB_TOKEN \
      --form ref=main \
      --form variables[CHART_FILE]="$CHART_FILE" \
      --form variables[HELM_CHART_VERSION]="$HELM_CHART_VERSION" \
      --form variables[HELM_APP_VERSION]="$CI_COMMIT_TAG" \
      --form variables[TRIGGER_ID]="$CI_JOB_ID" \
        --url "https://gitlab.com/api/v4/projects/$HELM_REPO_PROJECT_ID/trigger/pipeline"
    - echo "Triggered pipeline to upload helm chart in project $HELM_REPO_PROJECT_ID"