Merge branch 'CactuseSecurity:develop' into develop

Author: SolidProgramming

.githooks/pre-commit

@@ -3,10 +3,20 @@ set -euo pipefail
 
 if [ -d ".venv/Scripts" ]; then
     VENV_BIN=".venv/Scripts/"
+  echo "Using virtual environment from .venv (Windows)"
+    activate_script=".venv/Scripts/activate"
 elif [ -d ".venv/bin" ]; then
+    echo "Using virtual environment from .venv (Unix)"
     VENV_BIN=".venv/bin/"
+    activate_script=".venv/bin/activate"
 else
+    echo "Warning: No virtual environment found. Assuming ruff and pyright are globally available."
     VENV_BIN=""
+    activate_script=""
+fi
+
+if [ -f "$activate_script" ]; then
+    source "$activate_script"
 fi
 
 RUFF_PATH=$VENV_BIN"ruff"

pyproject.toml

@@ -3,6 +3,9 @@ include = [
     "roles/importer/files/importer",
     "scripts"
 ]
+extraPaths = ["roles/importer/files/importer"]
+venv = ".venv"
+
 exclude = [
     "**/.*",
     "**/node_modules",
@@ -17,7 +20,6 @@ exclude = [
     "scripts/release_lock.py"    
 ]
 
-
 typeCheckingMode = "strict"
 reportMissingImports = false
 reportMissingTypeStubs = true

roles/api/tasks/hasura-install.yml

@@ -39,15 +39,43 @@
     group: "{{ postgres_group }}"
   become: true
 
+- name: check for existing hasura admin secret file
+  stat:
+    path: "{{ fworch_secrets_dir }}/hasura_admin_pwd"
+  register: hasura_admin_secret_file
+  become: true
+
+- name: read existing hasura admin secret during upgrade
+  slurp:
+    src: "{{ fworch_secrets_dir }}/hasura_admin_pwd"
+  register: existing_hasura_admin_secret
+  become: true
+  when:
+    - installation_mode == "upgrade"
+    - hasura_admin_secret_file.stat.exists
+    - api_use_existing_hasura_on_upgrade | default(false) | bool
+
+- name: set hasura admin secret from existing file
+  set_fact:
+    api_hasura_admin_secret: "{{ existing_hasura_admin_secret['content'] | b64decode | trim }}"
+  when:
+    - installation_mode == "upgrade"
+    - hasura_admin_secret_file.stat.exists
+    - api_use_existing_hasura_on_upgrade | default(false) | bool
+
 - name: set static hasura admin pwd for test purposes only
   set_fact:
     api_hasura_admin_secret: "{{ api_hasura_admin_test_password }}"
-  when: testkeys is defined and testkeys|bool
+  when:
+    - api_hasura_admin_secret is not defined
+    - testkeys is defined
+    - testkeys | bool
 
 - name: set random hasura admin password
   set_fact:
     api_hasura_admin_secret: "{{ randomly_generated_pwd }}"
-  when: testkeys is not defined or not testkeys|bool
+  when:
+    - api_hasura_admin_secret is not defined
 
 - name: write hasura admin password to secrets directory
   copy:
@@ -57,6 +85,8 @@
     owner: "{{ fworch_user }}"
     group: "{{ fworch_group }}"
   become: true
+  when:
+    - installation_mode != "upgrade" or not (api_use_existing_hasura_on_upgrade | default(false) | bool) or not hasura_admin_secret_file.stat.exists
 
 - name: check for existing hasura cli file
   stat:
@@ -81,7 +111,9 @@
   retries: 3
   delay: 5
   until: hasura_release.status | default(-1) == 200
-  when: not api_cli_check.stat.exists
+  when:
+    - not api_cli_check.stat.exists
+    - not api_use_existing_hasura_on_upgrade | default(false) | bool
 
 - name: Extract Hasura CLI asset id for {{ linux_architecture }}
   set_fact:
@@ -95,6 +127,7 @@
       }}
   when:
     - not api_cli_check.stat.exists
+    - not api_use_existing_hasura_on_upgrade | default(false) | bool
     - hasura_release.status | default(-1) == 200
 
 - name: download {{ api_hasura_version }} hasura cli binary via authenticated GitHub access
@@ -114,6 +147,7 @@
   become: true
   when:
     - not api_cli_check.stat.exists
+    - not api_use_existing_hasura_on_upgrade | default(false) | bool
     - hasura_cli_asset_id is defined
 
 - name: download {{ api_hasura_version }} hasura cli binary via direct GitHub download
@@ -132,6 +166,7 @@
   become: true
   when:
     - not api_cli_check.stat.exists
+    - not api_use_existing_hasura_on_upgrade | default(false) | bool
     - hasura_cli_asset_id is not defined
 
 - name: initialize hasura cli directory
@@ -141,7 +176,9 @@
   become: true
   become_user: "{{ fworch_user }}"
   environment: "{{ proxy_env }}"
-  when: not api_cli_check.stat.exists
+  when:
+    - not api_cli_check.stat.exists
+    - not api_use_existing_hasura_on_upgrade | default(false) | bool
 
 - name: set hasura env variable
   set_fact:
@@ -177,6 +214,22 @@
     var: hasura_env
   when: debug_level > '1'
 
+- name: get existing hasura container info
+  docker_container_info:
+    name: "{{ api_container_name }}"
+  register: existing_hasura_container_info
+  become: true
+  become_user: "{{ fworch_user }}"
+
+- name: set hasura container reuse mode
+  set_fact:
+    api_reuse_existing_hasura_container: >-
+      {{
+        installation_mode == "upgrade"
+        and (api_use_existing_hasura_on_upgrade | default(false) | bool)
+        and (existing_hasura_container_info.exists | default(false) | bool)
+      }}
+
 - name: request Docker Hub token for hasura/graphql-engine
   uri:
     url: "https://auth.docker.io/token?service=registry.docker.io&scope=repository:hasura/graphql-engine:pull"
@@ -185,6 +238,7 @@
   register: dockerhub_token
   environment: "{{ proxy_env }}"
   failed_when: dockerhub_token.status | default(-1) != 200
+  when: not api_reuse_existing_hasura_container | bool
 
 - name: check Docker Hub manifest access for hasura/graphql-engine:{{ api_hasura_version }}
   uri:
@@ -198,6 +252,7 @@
   register: dockerhub_manifest_check
   environment: "{{ proxy_env }}"
   failed_when: false
+  when: not api_reuse_existing_hasura_container | bool
 
 - name: fail if Docker Hub manifest access is blocked
   fail:
@@ -206,7 +261,20 @@
       (HTTP {{ dockerhub_manifest_check.status | default('unknown') }}). This typically indicates
       blocked registry access or proxy restrictions. Ensure the host can reach registry-1.docker.io
       or configure a registry mirror.
-  when: dockerhub_manifest_check.status | default(0) != 200
+  when:
+    - not api_reuse_existing_hasura_container | bool
+    - dockerhub_manifest_check.status | default(0) != 200
+
+- name: start existing hasura container during upgrade
+  docker_container:
+    name: "{{ api_container_name }}"
+    state: started
+    container_default_behavior: no_defaults
+  register: docker_return
+  become: true
+  become_user: "{{ fworch_user }}"
+  environment: "{{ proxy_env }}"
+  when: api_reuse_existing_hasura_container | bool
 
 - name: start hasura container
   docker_container:
@@ -230,6 +298,7 @@
   become: true
   become_user: "{{ fworch_user }}"
   environment: "{{ proxy_env }}"
+  when: not api_reuse_existing_hasura_container | bool
 
 - name: show docker result
   debug:

roles/api/tasks/main.yml

@@ -8,6 +8,42 @@
   - api handler
   when: installation_mode == "upgrade"
 
+- name: set default hasura upgrade mode for upgrades
+  set_fact:
+    api_use_existing_hasura_on_upgrade: "{{ api_keep_existing_hasura_on_upgrade | default(false) | bool }}"
+  when: installation_mode == "upgrade"
+
+- name: probe GitHub release access for hasura upgrade
+  uri:
+    url: "https://api.github.com/repos/hasura/graphql-engine/releases/tags/{{ api_hasura_version }}"
+    method: GET
+    return_content: false
+  register: api_hasura_release_access
+  environment: "{{ proxy_env }}"
+  failed_when: false
+  when:
+    - installation_mode == "upgrade"
+    - not api_use_existing_hasura_on_upgrade | bool
+
+- name: fallback to existing hasura when GitHub is not reachable during upgrade
+  set_fact:
+    api_use_existing_hasura_on_upgrade: true
+  when:
+    - installation_mode == "upgrade"
+    - not api_use_existing_hasura_on_upgrade | bool
+    - api_hasura_release_access.status | default(0) != 200
+
+- name: show hasura upgrade fallback decision
+  debug:
+    msg: >-
+      GitHub release lookup failed (HTTP {{ api_hasura_release_access.status | default('unknown') }}),
+      reusing existing Hasura container and upgrading metadata only.
+  when:
+    - installation_mode == "upgrade"
+    - api_use_existing_hasura_on_upgrade | bool
+    - api_hasura_release_access is defined
+    - api_hasura_release_access.status | default(0) != 200
+
 - name: stop MW for upgrading
   ansible.builtin.systemd:
     name: "{{ product_name }}-middleware"
@@ -20,7 +56,7 @@
     name: "{{ api_service_name }}"
     state: stopped
   become: true
-  when: installation_mode == "upgrade"
+  when: installation_mode == "upgrade" and not (api_use_existing_hasura_on_upgrade | default(false) | bool)
   failed_when: false
 
 - name: stop API container for upgrading
@@ -30,7 +66,7 @@
     container_default_behavior: no_defaults
   become: true
   become_user: "{{ fworch_user }}"
-  when: installation_mode == "upgrade"
+  when: installation_mode == "upgrade" and not (api_use_existing_hasura_on_upgrade | default(false) | bool)
   failed_when: false
 
 - name: stop UI for upgrading
@@ -58,6 +94,7 @@
     path: "{{ api_home }}"
     state: absent
   become: true
+  when: installation_mode != "upgrade" or not (api_use_existing_hasura_on_upgrade | default(false) | bool)
 
 - name: create api home
   file: