mrc-6454 Replace Orderly Web with Packit

.github/workflows/test-and-push.yml

@@ -9,13 +9,13 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.12
 
       - name: Install dependencies
         run: |
-          pip3 install pytest-cov pycodestyle codecov
+          pip3 install pytest-cov pycodestyle codecov hatch
           pip3 install -r requirements.txt
 
       - name: Run dependencies

README.md

@@ -13,12 +13,13 @@ execute tasks, coordinated by Celery. Celery supports various message brokers in
 Clone the repo anywhere and install dependencies with (from the repo root):
 
 ```
-pip3 install --user -r requirements.txt
+hatch shell
+pip3 install -r requirements.txt
 ```
 
 ## Development
 
-Run dependencies (Montagu API and DB, OrderlyWeb and a local Redis message queue in docker) with `scripts/run-dependencies.sh`
+Run dependencies (Montagu API and DB, Proxy, Packit and a local Redis message queue in docker) with `scripts/run-dev-dependencies.sh`
 
 Dependencies also include a fake smtp server run from a [docker image](https://hub.docker.com/r/reachfive/fake-smtp-server)
 to enable development and testing of email functionality. You can see a web front end for the emails 'sent' via this server
@@ -54,7 +55,7 @@ The worker expects to find a config file at `config/config.yml`.
 
 The Dockerfile copies `config/docker_config.yml` to `config/config.yml`.
 This allows the worker running on metal to use a broker on `localhost` while the worker in docker needs to use
-`montagu_mq`, the container name of the broker, to access its port. 
+`montagu-mq`, the container name of the broker, to access its port. 
 
 Note that if a YouTrack token is not provided in the config the app will look for an environment variable called `YOUTRACK_TOKEN`. 
 This makes local and automated testing of the YouTrack integration possible. 

config/config.yml

@@ -4,8 +4,9 @@ servers:
     url: http://localhost:8080
     user: test.user@example.com
     password: password
-  orderlyweb:
-    url: http://localhost:8888
+  packit:
+    url: https://localhost/packit
+    disable_certificate_verify: true
   youtrack:
     token: None
   smtp:
@@ -27,17 +28,22 @@ tasks:
                 - minimal_modeller@example.com
                 - science@example.com
               subject: "VIMC diagnostic report: {touchstone} - {group} - {disease}"
-            timeout: 300
             assignee: a.hill
+            publish_roles:
+              - minimal.modeller
+              - Funders
           - report_name: diagnostic-param
             parameters:
-              nmin: 0
+              a: 1
+              b: 2
+              c: 3
             success_email:
               recipients:
                 - other_modeller@example.com
                 - science@example.com
               subject: "New version of another Orderly report"
-            timeout: 1200
             assignee: e.russell
+            publish_roles:
+              - other.modeller
   archive_folder_contents:
     min_file_age_seconds: 0

config/docker_config.yml

@@ -1,15 +1,16 @@
-host: montagu_mq_1
+host: montagu-mq-1
 servers:
   montagu:
-    url: http://montagu_api_1:8080
+    url: http://montagu-api-1:8080
     user: test.user@example.com
     password: password
-  orderlyweb:
-    url: http://montagu_orderly_web:8888
+  packit:
+    url: https://reverse-proxy:443/packit
+    disable_certificate_verify: true
   youtrack:
     token:
   smtp:
-    host: montagu_smtp_server_1
+    host: montagu-smtp_server-1
     port: 1025
     from: noreply@example.com
 tasks:
@@ -26,14 +27,21 @@ tasks:
                 - science@example.com
               subject:  "VIMC diagnostic report: {touchstone} - {group} - {disease}"
             assignee: a.hill
+            publish_roles:
+              - minimal.modeller
+              - Funders
           - report_name: diagnostic-param
             parameters:
-              nmin: 0
+              a: 1
+              b: 2
+              c: 3
             success_email:
               recipients:
                 - other_modeller@example.com
                 - science@example.com
               subject: "New version of another Orderly report"
             assignee: e.russell
+            publish_roles:
+              - other.modeller
   archive_folder_contents:
     min_file_age_seconds: 0

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   mq:
     image: redis
@@ -9,8 +8,8 @@ services:
     ports:
       - "5555:5555"
     environment:
-      - CELERY_BROKER_URL=redis://montagu_mq_1//
-      - CELERY_RESULT_BACKEND=redis://montagu_mq_1/0
+      - CELERY_BROKER_URL=redis://montagu-mq-1//
+      - CELERY_RESULT_BACKEND=redis://montagu-mq-1/0
       - FLOWER_PORT=5555
   api:
     image: ${REGISTRY}/montagu-api:master
@@ -23,8 +22,16 @@ services:
     ports:
       - "5432:5432"
     command: /etc/montagu/postgresql.test.conf
+  contrib:
+    image: ${REGISTRY}/montagu-contrib-portal:master
+    depends_on:
+      - api
+  admin:
+      image: ${REGISTRY}/montagu-admin-portal:master
+      depends_on:
+        - api
   smtp_server:
     image: reachfive/fake-smtp-server
     ports:
       - "1025:1025"
-      - "1080:1080"
+      - "1080:1080"

requirements.txt

@@ -2,6 +2,6 @@ celery[redis]
 future
 pyyaml
 montagu>=0.0.2
-orderlyweb-api>=1.0.0
 git+https://github.com/reside-ic/youtrack-rest-python-library
 constellation
+requests-mock

scripts/clear-docker.sh

@@ -1,3 +1,5 @@
+set -x
+
 docker stop $(docker ps -aq)
 docker rm $(docker ps -aq)
 docker network prune --force

scripts/packit.yml

@@ -0,0 +1,50 @@
+container_prefix: montagu
+
+protect_data: false
+
+repo: ghcr.io/mrc-ide
+
+network: montagu_default
+
+volumes:
+  outpack: montagu_outpack_volume
+  packit_db: montagu_packit_db
+  orderly_library: montagu_orderly_library
+  orderly_logs: montagu_orderly_logs
+
+outpack:
+  server:
+    name: outpack_server
+    tag: main
+  migrate:
+    name: outpack.orderly
+    tag: main
+
+packit:
+  base_url: https://localhost
+  api:
+    name: packit-api
+    tag: main
+  app:
+    name: montagu-packit
+    tag: main
+  db:
+    name: packit-db
+    tag: main
+    user: packituser
+    password: changeme
+  auth:
+    enabled: true
+    auth_method: preauth
+    # We'll get this from the vault on production
+    jwt:
+      secret: "0b4g4f8z4mdsrhoxfde2mam8f00vmt0f"
+    expiry_days: 1
+
+orderly-runner:
+  image:
+    name: orderly.runner
+    tag: main
+  git:
+    url: https://github.com/reside-ic/orderly2-example.git
+  workers: 1

scripts/run-dependencies.sh

@@ -1,32 +1,27 @@
 #!/usr/bin/env bash
 set -ex
 
-here=$(dirname $0)
+export REGISTRY=vimc
 
+here=$(dirname $0)
 ./scripts/clear-docker.sh
 docker network prune -f
 
-export REGISTRY=vimc
 export NETWORK=montagu_default
 
 # Run the API and database
-docker-compose pull
-docker-compose --project-name montagu up -d
+docker compose pull
+docker compose --project-name montagu up -d
 
 # Clear redis
-docker exec montagu_mq_1 redis-cli FLUSHALL
-
-# Install orderly-web
-pip3 install constellation
-pip3 install orderly-web
-orderly-web start $here
+docker exec montagu-mq-1 redis-cli FLUSHALL
 
 # Start the APIs
-docker exec montagu_api_1 mkdir -p /etc/montagu/api/
-docker exec montagu_api_1 touch /etc/montagu/api/go_signal
+docker exec montagu-api-1 mkdir -p /etc/montagu/api/
+docker exec montagu-api-1 touch /etc/montagu/api/go_signal
 
 # Wait for the database
-docker exec montagu_db_1 montagu-wait.sh
+docker exec montagu-db-1 montagu-wait.sh
 
 # migrate the database
 migrate_image=${REGISTRY}/montagu-migrate:master
@@ -40,6 +35,41 @@ $here/montagu_cli.sh add "Test User" test.user \
 
 $here/montagu_cli.sh addRole test.user user
 
-# Add user to orderlyweb
-$here/orderlyweb_cli.sh add-users test.user@example.com
-$here/orderlyweb_cli.sh grant test.user@example.com */reports.read */reports.run */reports.review
+# Run packit
+hatch env run pip3 install constellation
+hatch env run pip3 install packit-deploy
+# For some reason packit is emitting exit code 1 despite apparently succeeding. Allow this for now...
+set +e
+hatch env run -- packit start --pull $here
+echo Packit deployed with exit code $?
+set -e
+
+docker exec montagu-packit-db wait-for-db
+
+# Run the proxy here, not through docker compose - it needs packit to be running before it will start up
+MONTAGU_PROXY_TAG=vimc/montagu-reverse-proxy:master
+docker pull $MONTAGU_PROXY_TAG
+docker run -d \
+  -p "443:443" -p "80:80" \
+	--name reverse-proxy \
+	--network montagu_default\
+	$MONTAGU_PROXY_TAG 443 localhost
+
+# give packit api some time to migrate the db...
+sleep 5
+
+# create roles to publish to...
+docker exec -i montagu-packit-db psql -U packituser -d packit --single-transaction <<EOF
+insert into "role" (name, is_username) values ('Funders', FALSE);
+insert into "role" (name, is_username) values ('minimal.modeller', FALSE);
+insert into "role" (name, is_username) values ('other.modeller', FALSE);
+EOF
+
+# Add user to packit, as admin
+USERNAME='test.user'
+EMAIL='test.user@example.com'
+DISPLAY_NAME='Test User'
+ROLE='ADMIN'
+docker exec montagu-packit-db create-preauth-user --username "$USERNAME" --email "$EMAIL" --displayname "$DISPLAY_NAME" --role "$ROLE"
+
+

scripts/run-dev-dependencies.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -ex
+
+./scripts/run-dependencies.sh
+
+# From now on, if the user presses Ctrl+C we should teardown gracefully
+function cleanup() {
+  docker container stop reverse-proxy
+  docker container rm reverse-proxy -v
+  # Same exit code issue for packit stop as packit start....
+  set +e
+  hatch env run -- packit stop ./scripts
+  set -e
+  # remove db volume manually rather than --volumes flag to packit, to avoid requiring user confirmation
+  docker volume rm montagu_packit_db montagu_orderly_library montagu_outpack_volume montagu_orderly_logs
+  docker compose --project-name montagu down -v
+}
+trap cleanup EXIT
+
+# Wait for Ctrl+C
+echo "Ready to use. Press Ctrl+C to teardown."
+sleep infinity