ci: add opt-in tenant-parallel BVT execution

.github/workflows/e2e-compose-parallel.yaml

@@ -13,6 +13,16 @@ on:
         required: false
         type: string
         default: ""
+      tenant_parallel_enabled:
+        description: "Run directory-safe BVT cases in tenant workers"
+        required: false
+        type: boolean
+        default: false
+      ci_ref:
+        description: "matrixorigin/CI ref containing tenant BVT scripts"
+        required: false
+        type: string
+        default: "main"
     secrets:
       TOKEN_ACTION:
         description: "Token for checkout (e.g. pull from fork/private)"
@@ -210,7 +220,7 @@ jobs:
       - name: checkout
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
-          token: ${{ secrets.TOKEN_ACTION }}
+          token: ${{ secrets.TOKEN_ACTION || github.token }}
           fetch-depth: "3"
           repository: ${{ github.event.pull_request.head.repo.full_name }}
           ref: ${{ github.event.pull_request.head.sha }}
@@ -289,6 +299,14 @@ jobs:
           path: ./mo-tester
           ref: main
 
+      - name: Checkout tenant-parallel BVT scripts
+        if: ${{ inputs.tenant_parallel_enabled }}
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          repository: matrixorigin/CI
+          ref: ${{ inputs.ci_ref }}
+          path: .ci/tenant-parallel
+
       - name: Update BVT SQL Timeout
         run: |
           cd $GITHUB_WORKSPACE/mo-tester
@@ -341,10 +359,19 @@ jobs:
               "https://raw.githubusercontent.com/matrixorigin/matrixone/7cb78aa5f88e3b98163f6b3bffb5f71586c0cbbf/optools/run_bvt_group.sh" \
               --output "${bvt_runner}"
           fi
-          bash "${bvt_runner}" \
-            "$GITHUB_WORKSPACE/mo-tester" \
-            "$GITHUB_WORKSPACE/test/distributed/cases" \
-            "${bvt_group}" 2>&1 | tee "${RUNNER_TEMP}/bvt-compose.log"
+          if [ '${{ inputs.tenant_parallel_enabled }}' = 'true' ]; then
+            bash "$GITHUB_WORKSPACE/.ci/tenant-parallel/scripts/run_bvt_tenant_parallel.sh" \
+              --tester-dir "$GITHUB_WORKSPACE/mo-tester" \
+              --case-root "$GITHUB_WORKSPACE/test/distributed/cases" \
+              --group "${bvt_group}" \
+              --directories "$GITHUB_WORKSPACE/.ci/tenant-parallel/scripts/bvt_tenant_directories.sh" \
+              --output-dir "${RUNNER_TEMP}/bvt-tenant-compose"
+          else
+            bash "${bvt_runner}" \
+              "$GITHUB_WORKSPACE/mo-tester" \
+              "$GITHUB_WORKSPACE/test/distributed/cases" \
+              "${bvt_group}"
+          fi 2>&1 | tee "${RUNNER_TEMP}/bvt-compose.log"
 
       - name: Summarize Compose BVT result
         if: ${{ always() && !cancelled() }}
@@ -359,12 +386,24 @@ jobs:
           fi
           {
             echo '### Compose + Proxy BVT result'
-            grep '^Run BVT group' "${report}" | sed 's/^/- /' || true
+            grep -E '^(Run BVT group|BVT directory plan:)' "${report}" | sed 's/^/- /' || true
             echo
             echo '#### Top 10 slow BVT scripts'
             echo
           } | tee -a "$GITHUB_STEP_SUMMARY"
 
+          tenant_summary="${RUNNER_TEMP}/bvt-tenant-compose/summary.tsv"
+          if [ -s "${tenant_summary}" ]; then
+            {
+              echo '#### Tenant BVT phase results'
+              echo
+              echo '```text'
+              cat "${tenant_summary}"
+              echo '```'
+              echo
+            } | tee -a "$GITHUB_STEP_SUMMARY"
+          fi
+
           sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \
             | sort -nr -k1,1 | awk 'NR <= 10 { print }' > "${timing}"
           if [ ! -s "${timing}" ]; then
@@ -382,7 +421,9 @@ jobs:
         uses: actions/upload-artifact@v7
         with:
           name: bvt-execution-compose
-          path: ${{ runner.temp }}/bvt-compose.log
+          path: |
+            ${{ runner.temp }}/bvt-compose.log
+            ${{ runner.temp }}/bvt-tenant-compose
           if-no-files-found: warn
           retention-days: 7
 

.github/workflows/e2e-standalone-parallel.yaml

@@ -13,6 +13,16 @@ on:
         required: false
         type: string
         default: ""
+      tenant_parallel_enabled:
+        description: "Run directory-safe BVT cases in tenant workers"
+        required: false
+        type: boolean
+        default: false
+      ci_ref:
+        description: "matrixorigin/CI ref containing tenant BVT scripts"
+        required: false
+        type: string
+        default: "main"
     secrets:
       TOKEN_ACTION:
         description: "Token for checkout (e.g. pull from fork/private)"
@@ -258,7 +268,7 @@ jobs:
       - name: checkout head
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
-          token: ${{ secrets.TOKEN_ACTION }}
+          token: ${{ secrets.TOKEN_ACTION || github.token }}
           fetch-depth: "3"
           path: ./head
           repository: ${{ github.event.pull_request.head.repo.full_name }}
@@ -302,6 +312,13 @@ jobs:
           repository: matrixorigin/mo-tester
           path: ./mo-tester
           ref: main
+      - name: Checkout tenant-parallel BVT scripts
+        if: ${{ inputs.tenant_parallel_enabled }}
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          repository: matrixorigin/CI
+          ref: ${{ inputs.ci_ref }}
+          path: .ci/tenant-parallel
       - name: Update BVT SQL Timeout
         run: |
           cd $GITHUB_WORKSPACE/mo-tester
@@ -353,12 +370,21 @@ jobs:
               "https://raw.githubusercontent.com/matrixorigin/matrixone/7cb78aa5f88e3b98163f6b3bffb5f71586c0cbbf/optools/run_bvt_group.sh" \
               --output "${bvt_runner}"
           fi
-          bash "${bvt_runner}" \
-            "$GITHUB_WORKSPACE/mo-tester" \
-            "$GITHUB_WORKSPACE/head/test/distributed/cases" \
-            "${bvt_group}" \
-            "$GITHUB_WORKSPACE/head/test/distributed/resources" \
-            2>&1 | tee "${RUNNER_TEMP}/bvt-pessimistic.log"
+          if [ '${{ inputs.tenant_parallel_enabled }}' = 'true' ]; then
+            bash "$GITHUB_WORKSPACE/.ci/tenant-parallel/scripts/run_bvt_tenant_parallel.sh" \
+              --tester-dir "$GITHUB_WORKSPACE/mo-tester" \
+              --case-root "$GITHUB_WORKSPACE/head/test/distributed/cases" \
+              --group "${bvt_group}" \
+              --directories "$GITHUB_WORKSPACE/.ci/tenant-parallel/scripts/bvt_tenant_directories.sh" \
+              --output-dir "${RUNNER_TEMP}/bvt-tenant-pessimistic" \
+              --resource-dir "$GITHUB_WORKSPACE/head/test/distributed/resources"
+          else
+            bash "${bvt_runner}" \
+              "$GITHUB_WORKSPACE/mo-tester" \
+              "$GITHUB_WORKSPACE/head/test/distributed/cases" \
+              "${bvt_group}" \
+              "$GITHUB_WORKSPACE/head/test/distributed/resources"
+          fi 2>&1 | tee "${RUNNER_TEMP}/bvt-pessimistic.log"
 
       - name: Summarize Launch + Pessimistic BVT result
         if: ${{ always() && !cancelled() }}
@@ -373,12 +399,24 @@ jobs:
           fi
           {
             echo '### Launch + Pessimistic BVT result'
-            grep '^Run BVT group' "${report}" | sed 's/^/- /' || true
+            grep -E '^(Run BVT group|BVT directory plan:)' "${report}" | sed 's/^/- /' || true
             echo
             echo '#### Top 10 slow BVT scripts'
             echo
           } | tee -a "$GITHUB_STEP_SUMMARY"
 
+          tenant_summary="${RUNNER_TEMP}/bvt-tenant-pessimistic/summary.tsv"
+          if [ -s "${tenant_summary}" ]; then
+            {
+              echo '#### Tenant BVT phase results'
+              echo
+              echo '```text'
+              cat "${tenant_summary}"
+              echo '```'
+              echo
+            } | tee -a "$GITHUB_STEP_SUMMARY"
+          fi
+
           sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \
             | sort -nr -k1,1 | awk 'NR <= 10 { print }' > "${timing}"
           if [ ! -s "${timing}" ]; then
@@ -396,7 +434,9 @@ jobs:
         uses: actions/upload-artifact@v7
         with:
           name: bvt-execution-pessimistic
-          path: ${{ runner.temp }}/bvt-pessimistic.log
+          path: |
+            ${{ runner.temp }}/bvt-pessimistic.log
+            ${{ runner.temp }}/bvt-tenant-pessimistic
           if-no-files-found: warn
           retention-days: 7
 

scripts/bvt_tenant_directories.sh

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+
+# Top-level MatrixOne BVT case directories are the scheduling unit. This file is
+# sourced by run_bvt_tenant_parallel.sh after bvt_group has been validated.
+
+if ! [[ "${bvt_group:-}" =~ ^[01]$ ]]; then
+  echo "bvt_tenant_directories.sh: bvt_group must be 0 or 1" >&2
+  return 2 2>/dev/null || exit 2
+fi
+
+bvt_group_0_directories=(
+  array auto_increment benchmark dataXtest ddl disttae fake_pk function
+  git4data hint join keyword load_data mo_cloud optimizer pg_cast plugin
+  prepare procedure query_result sample save_query_result sequence snapshot
+  sql_inject stage system system_variable temporary tenant tenxcloud_xx
+  time_window union util vector view
+)
+
+bvt_group_1_directories=(
+  analyze charset_collation comment cte database distinct dml dtype expression
+  feature_limit foreign_key fulltext geo iceberg log metadata operator
+  pessimistic_transaction pitr plan_cache publication_subscription qexec
+  recursive_cte replace_statement result_count security set sql_source_type
+  statement_query_type subquery table task udf window zz_accesscontrol
+  zz_statement_query_type
+)
+
+bvt_serial_before_all=(
+  result_count
+  sql_source_type
+  statement_query_type
+)
+
+bvt_parallel_group_0_worker_0=(
+  view
+)
+
+bvt_parallel_group_0_worker_1=(
+  auto_increment
+  sequence
+  procedure
+  keyword
+  sample
+  pg_cast
+  plugin
+  time_window
+  union
+  fake_pk
+  dataXtest
+)
+
+bvt_parallel_group_1_worker_0=(
+  dtype
+  expression
+  comment
+  recursive_cte
+  qexec
+  replace_statement
+)
+
+bvt_parallel_group_1_worker_1=(
+  window
+  fulltext
+  operator
+  geo
+  charset_collation
+  distinct
+  udf
+  cte
+  plan_cache
+)
+
+bvt_serial_after_all=(
+  log
+  analyze
+  array
+  benchmark
+  database
+  ddl
+  disttae
+  dml
+  feature_limit
+  foreign_key
+  function
+  git4data
+  hint
+  iceberg
+  join
+  load_data
+  metadata
+  mo_cloud
+  optimizer
+  pessimistic_transaction
+  pitr
+  prepare
+  publication_subscription
+  query_result
+  save_query_result
+  security
+  set
+  snapshot
+  sql_inject
+  stage
+  subquery
+  system
+  system_variable
+  table
+  task
+  temporary
+  tenant
+  tenxcloud_xx
+  util
+  vector
+  zz_accesscontrol
+  zz_statement_query_type
+)
+
+bvt_excluded=(
+  optimistic
+)
+
+bvt_directory_in_array() {
+  local expected=$1
+  shift
+  local value
+  for value in "$@"; do
+    if [[ "${value}" == "${expected}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+bvt_group_for_directory() {
+  local name=$1
+  if bvt_directory_in_array "${name}" "${bvt_group_0_directories[@]}"; then
+    printf '0\n'
+  elif bvt_directory_in_array "${name}" "${bvt_group_1_directories[@]}"; then
+    printf '1\n'
+  else
+    printf '%s' "${name}" |
+      cksum |
+      awk '{ print $1 % 2 }'
+  fi
+}
+
+bvt_serial_before=()
+bvt_serial_after=()
+bvt_worker_0=()
+bvt_worker_1=()
+
+for bvt_directory_name in "${bvt_serial_before_all[@]}"; do
+  if [[ "$(bvt_group_for_directory "${bvt_directory_name}")" == "${bvt_group}" ]]; then
+    bvt_serial_before+=("${bvt_directory_name}")
+  fi
+done
+
+for bvt_directory_name in "${bvt_serial_after_all[@]}"; do
+  if [[ "$(bvt_group_for_directory "${bvt_directory_name}")" == "${bvt_group}" ]]; then
+    bvt_serial_after+=("${bvt_directory_name}")
+  fi
+done
+
+if [[ "${bvt_group}" == "0" ]]; then
+  bvt_worker_0=("${bvt_parallel_group_0_worker_0[@]}")
+  bvt_worker_1=("${bvt_parallel_group_0_worker_1[@]}")
+else
+  bvt_worker_0=("${bvt_parallel_group_1_worker_0[@]}")
+  bvt_worker_1=("${bvt_parallel_group_1_worker_1[@]}")
+fi
+
+unset bvt_directory_name