From 3c729d638ff47c982efbb00ea87a7be4cfdf55b3 Mon Sep 17 00:00:00 2001 From: Adam Saghy Date: Thu, 21 May 2026 18:11:00 +0100 Subject: [PATCH] FINERACT-2421: Increase matrix job shard number --- .github/workflows/build-e2e-tests.yml | 4 +- .github/workflows/build-mariadb.yml | 68 +++++++++---------- .github/workflows/build-mysql.yml | 68 +++++++++---------- .github/workflows/build-postgresql.yml | 68 +++++++++---------- .../factory/LoanProductsRequestFactory.java | 20 +++--- 5 files changed, 106 insertions(+), 122 deletions(-) diff --git a/.github/workflows/build-e2e-tests.yml b/.github/workflows/build-e2e-tests.yml index c5e8a11faa4..77df4082caa 100644 --- a/.github/workflows/build-e2e-tests.yml +++ b/.github/workflows/build-e2e-tests.yml @@ -16,8 +16,8 @@ jobs: fail-fast: false matrix: # Define the number of shards (1-based indexing) - shard_index: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] - total_shards: [15] + shard_index: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] + total_shards: [20] env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} diff --git a/.github/workflows/build-mariadb.yml b/.github/workflows/build-mariadb.yml index f9f813d5bab..b5089e199f8 100644 --- a/.github/workflows/build-mariadb.yml +++ b/.github/workflows/build-mariadb.yml @@ -8,13 +8,16 @@ permissions: jobs: test: + name: MariaDB Tests (Shard ${{ matrix.shard_index }} of ${{ matrix.total_shards }}) runs-on: ubuntu-24.04 timeout-minutes: 60 strategy: fail-fast: false matrix: - task: [test-core-1, test-core-2, test-core-3, test-core-4, test-core-5] + # Define the number of shards (1-based indexing) + shard_index: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ] + total_shards: [ 15 ] services: mariadb: @@ -86,47 +89,40 @@ jobs: sleep 10 docker exec localstack awslocal s3api create-bucket --bucket fineract-reports - - name: Generate test class list (only for test-core-X) - if: startsWith(matrix.task, 'test-core-') + - name: Generate test class list run: | chmod +x scripts/split-tests.sh - SHARD_INDEX=$(echo "${{ matrix.task }}" | awk -F'-' '{print $3}') - ./scripts/split-tests.sh 5 $SHARD_INDEX - cat "shard-tests_${SHARD_INDEX}.txt" + ./scripts/split-tests.sh ${{ matrix.total_shards }} ${{ matrix.shard_index }} + cat "shard-tests_${{ matrix.shard_index }}.txt" - name: Run Gradle Task run: | set -e # Fail the script if any command fails - SHARD_INDEX=$(echo "${{ matrix.task }}" | awk -F'-' '{print $3}') FAILED=0 - case "${{ matrix.task }}" in - test-core-*) - echo "Grouping test classes by module..." - declare -A module_tests - - while IFS=, read -r module class; do - module_tests["$module"]+="$class " - done < "shard-tests_${SHARD_INDEX}.txt" - - for module in "${!module_tests[@]}"; do - echo "::group::Running tests in $module" - for class in ${module_tests[$module]}; do - echo " - $class" - done - - # Build test args - test_args=$(for class in ${module_tests[$module]}; do echo --tests "$class"; done | xargs) - - # Run test task for this module - if ! ./gradlew --no-daemon "$module:test" $test_args -x checkstyleJmh -x checkstyleMain -x checkstyleTest -x spotlessCheck -x spotlessApply -x spotbugsMain -x spotbugsTest -x javadoc -x javadocJar -x modernizer -x buildJavaSdk ; then - echo "::error::Tests failed in module $module" - FAILED=1 - fi - echo "::endgroup::" - done - ;; - esac + echo "Grouping test classes by module..." + declare -A module_tests + + while IFS=, read -r module class; do + module_tests["$module"]+="$class " + done < "shard-tests_${{ matrix.shard_index }}.txt" + + for module in "${!module_tests[@]}"; do + echo "::group::Running tests in $module" + for class in ${module_tests[$module]}; do + echo " - $class" + done + + # Build test args + test_args=$(for class in ${module_tests[$module]}; do echo --tests "$class"; done | xargs) + + # Run test task for this module + if ! ./gradlew --no-daemon "$module:test" $test_args -x checkstyleJmh -x checkstyleMain -x checkstyleTest -x spotlessCheck -x spotlessApply -x spotbugsMain -x spotbugsTest -x javadoc -x javadocJar -x modernizer -x buildJavaSdk ; then + echo "::error::Tests failed in module $module" + FAILED=1 + fi + echo "::endgroup::" + done # Exit with failure status if any test failed if [ "$FAILED" -eq 1 ]; then @@ -138,7 +134,7 @@ jobs: if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: test-results-mariadb-${{ matrix.task }}-attempt-${{ github.run_attempt }} + name: test-results-mariadb-shard-${{ matrix.shard_index }}-attempt-${{ github.run_attempt }} path: '**/build/reports/' retention-days: 5 @@ -146,6 +142,6 @@ jobs: if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: server-logs-mariadb-${{ matrix.task }}-attempt-${{ github.run_attempt }} + name: server-logs-mariadb-shard-${{ matrix.shard_index }}-attempt-${{ github.run_attempt }} path: '**/build/cargo/' retention-days: 5 diff --git a/.github/workflows/build-mysql.yml b/.github/workflows/build-mysql.yml index 86fd8047c76..1577313add3 100644 --- a/.github/workflows/build-mysql.yml +++ b/.github/workflows/build-mysql.yml @@ -8,13 +8,16 @@ permissions: jobs: test: + name: MySQL Tests (Shard ${{ matrix.shard_index }} of ${{ matrix.total_shards }}) runs-on: ubuntu-24.04 timeout-minutes: 60 strategy: fail-fast: false matrix: - task: [test-core-1, test-core-2, test-core-3, test-core-4, test-core-5] + # Define the number of shards (1-based indexing) + shard_index: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ] + total_shards: [ 15 ] services: mysql: @@ -81,47 +84,40 @@ jobs: sleep 10 docker exec localstack awslocal s3api create-bucket --bucket fineract-reports - - name: Generate test class list (only for test-core-X) - if: startsWith(matrix.task, 'test-core-') + - name: Generate test class list run: | chmod +x scripts/split-tests.sh - SHARD_INDEX=$(echo "${{ matrix.task }}" | awk -F'-' '{print $3}') - ./scripts/split-tests.sh 5 $SHARD_INDEX - cat "shard-tests_${SHARD_INDEX}.txt" + ./scripts/split-tests.sh ${{ matrix.total_shards }} ${{ matrix.shard_index }} + cat "shard-tests_${{ matrix.shard_index }}.txt" - name: Run Gradle Task run: | set -e # Fail the script if any command fails - SHARD_INDEX=$(echo "${{ matrix.task }}" | awk -F'-' '{print $3}') FAILED=0 - case "${{ matrix.task }}" in - test-core-*) - echo "Grouping test classes by module..." - declare -A module_tests - - while IFS=, read -r module class; do - module_tests["$module"]+="$class " - done < "shard-tests_${SHARD_INDEX}.txt" - - for module in "${!module_tests[@]}"; do - echo "::group::Running tests in $module" - for class in ${module_tests[$module]}; do - echo " - $class" - done - - # Build test args - test_args=$(for class in ${module_tests[$module]}; do echo --tests "$class"; done | xargs) - - # Run test task for this module - if ! ./gradlew --no-daemon "$module:test" $test_args -PdbType=mysql -x checkstyleJmh -x checkstyleMain -x checkstyleTest -x spotlessCheck -x spotlessApply -x spotbugsMain -x spotbugsTest -x javadoc -x javadocJar -x modernizer -x buildJavaSdk ; then - echo "::error::Tests failed in module $module" - FAILED=1 - fi - echo "::endgroup::" - done - ;; - esac + echo "Grouping test classes by module..." + declare -A module_tests + + while IFS=, read -r module class; do + module_tests["$module"]+="$class " + done < "shard-tests_${{ matrix.shard_index }}.txt" + + for module in "${!module_tests[@]}"; do + echo "::group::Running tests in $module" + for class in ${module_tests[$module]}; do + echo " - $class" + done + + # Build test args + test_args=$(for class in ${module_tests[$module]}; do echo --tests "$class"; done | xargs) + + # Run test task for this module + if ! ./gradlew --no-daemon "$module:test" $test_args -PdbType=mysql -x checkstyleJmh -x checkstyleMain -x checkstyleTest -x spotlessCheck -x spotlessApply -x spotbugsMain -x spotbugsTest -x javadoc -x javadocJar -x modernizer -x buildJavaSdk ; then + echo "::error::Tests failed in module $module" + FAILED=1 + fi + echo "::endgroup::" + done # Exit with failure status if any test failed if [ "$FAILED" -eq 1 ]; then @@ -133,7 +129,7 @@ jobs: if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: test-results-mysql-${{ matrix.task }}-attempt-${{ github.run_attempt }} + name: test-results-mysql-shard-${{ matrix.shard_index }}-attempt-${{ github.run_attempt }} path: '**/build/reports/' retention-days: 5 @@ -141,6 +137,6 @@ jobs: if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: server-logs-mysql-${{ matrix.task }}-attempt-${{ github.run_attempt }} + name: server-logs-mysql-shard-${{ matrix.shard_index }}-attempt-${{ github.run_attempt }} path: '**/build/cargo/' retention-days: 5 diff --git a/.github/workflows/build-postgresql.yml b/.github/workflows/build-postgresql.yml index 4e850e15bf4..8598d4a96fa 100644 --- a/.github/workflows/build-postgresql.yml +++ b/.github/workflows/build-postgresql.yml @@ -8,13 +8,16 @@ permissions: jobs: test: + name: PostgreSQL Tests (Shard ${{ matrix.shard_index }} of ${{ matrix.total_shards }}) runs-on: ubuntu-24.04 timeout-minutes: 60 strategy: fail-fast: false matrix: - task: [test-core-1, test-core-2, test-core-3, test-core-4, test-core-5] + # Define the number of shards (1-based indexing) + shard_index: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ] + total_shards: [ 15 ] services: postgresql: @@ -82,47 +85,40 @@ jobs: sleep 10 docker exec localstack awslocal s3api create-bucket --bucket fineract-reports - - name: Generate test class list (only for test-core-X) - if: startsWith(matrix.task, 'test-core-') + - name: Generate test class list run: | chmod +x scripts/split-tests.sh - SHARD_INDEX=$(echo "${{ matrix.task }}" | awk -F'-' '{print $3}') - ./scripts/split-tests.sh 5 $SHARD_INDEX - cat "shard-tests_${SHARD_INDEX}.txt" + ./scripts/split-tests.sh ${{ matrix.total_shards }} ${{ matrix.shard_index }} + cat "shard-tests_${{ matrix.shard_index }}.txt" - name: Run Gradle Task run: | set -e # Fail the script if any command fails - SHARD_INDEX=$(echo "${{ matrix.task }}" | awk -F'-' '{print $3}') FAILED=0 - case "${{ matrix.task }}" in - test-core-*) - echo "Grouping test classes by module..." - declare -A module_tests - - while IFS=, read -r module class; do - module_tests["$module"]+="$class " - done < "shard-tests_${SHARD_INDEX}.txt" - - for module in "${!module_tests[@]}"; do - echo "::group::Running tests in $module" - for class in ${module_tests[$module]}; do - echo " - $class" - done - - # Build test args - test_args=$(for class in ${module_tests[$module]}; do echo --tests "$class"; done | xargs) - - # Run test task for this module - if ! ./gradlew --no-daemon "$module:test" $test_args -PdbType=postgresql -x checkstyleJmh -x checkstyleMain -x checkstyleTest -x spotlessCheck -x spotlessApply -x spotbugsMain -x spotbugsTest -x javadoc -x javadocJar -x modernizer -x buildJavaSdk ; then - echo "::error::Tests failed in module $module" - FAILED=1 - fi - echo "::endgroup::" - done - ;; - esac + echo "Grouping test classes by module..." + declare -A module_tests + + while IFS=, read -r module class; do + module_tests["$module"]+="$class " + done < "shard-tests_${{ matrix.shard_index }}.txt" + + for module in "${!module_tests[@]}"; do + echo "::group::Running tests in $module" + for class in ${module_tests[$module]}; do + echo " - $class" + done + + # Build test args + test_args=$(for class in ${module_tests[$module]}; do echo --tests "$class"; done | xargs) + + # Run test task for this module + if ! ./gradlew --no-daemon "$module:test" $test_args -PdbType=postgresql -x checkstyleJmh -x checkstyleMain -x checkstyleTest -x spotlessCheck -x spotlessApply -x spotbugsMain -x spotbugsTest -x javadoc -x javadocJar -x modernizer -x buildJavaSdk ; then + echo "::error::Tests failed in module $module" + FAILED=1 + fi + echo "::endgroup::" + done # Exit with failure status if any test failed if [ "$FAILED" -eq 1 ]; then @@ -134,7 +130,7 @@ jobs: if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: test-results-postgresql-${{ matrix.task }}-attempt-${{ github.run_attempt }} + name: test-results-postgresql-shard-${{ matrix.shard_index }}-attempt-${{ github.run_attempt }} path: '**/build/reports/' retention-days: 5 @@ -142,6 +138,6 @@ jobs: if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: server-logs-postgresql-${{ matrix.task }}-attempt-${{ github.run_attempt }} + name: server-logs-postgresql-shard-${{ matrix.shard_index }}-attempt-${{ github.run_attempt }} path: '**/build/cargo/' retention-days: 5 diff --git a/fineract-e2e-tests-core/src/test/java/org/apache/fineract/test/factory/LoanProductsRequestFactory.java b/fineract-e2e-tests-core/src/test/java/org/apache/fineract/test/factory/LoanProductsRequestFactory.java index 02a4732860b..d91464abf76 100644 --- a/fineract-e2e-tests-core/src/test/java/org/apache/fineract/test/factory/LoanProductsRequestFactory.java +++ b/fineract-e2e-tests-core/src/test/java/org/apache/fineract/test/factory/LoanProductsRequestFactory.java @@ -23,9 +23,9 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import lombok.RequiredArgsConstructor; import org.apache.fineract.client.models.AllowAttributeOverrides; import org.apache.fineract.client.models.GetLoanPaymentChannelToFundSourceMappings; @@ -72,7 +72,7 @@ public class LoanProductsRequestFactory { private final CodeValueResolver codeValueResolver; private final DelinquencyBucketResolver delinquencyBucketResolver; - private final Set productShortNameMap = new HashSet<>(); + private final Set productShortNameMap = ConcurrentHashMap.newKeySet(); @Autowired private CodeHelper codeHelper; @@ -1903,16 +1903,12 @@ public PostLoanProductsRequest defaultLoanProductsRequestLP2EmiUSD() { } public String generateShortNameSafely() { - String generatedShortName; - int counter = 0; - do { - counter++; - generatedShortName = Utils.randomStringGenerator("", 4); - if (counter > 999) { - throw new RuntimeException("Unable to generate unique short name"); + for (int counter = 0; counter < 999; counter++) { + String generatedShortName = Utils.randomStringGenerator("", 4); + if (productShortNameMap.add(generatedShortName)) { + return generatedShortName; } - } while (productShortNameMap.contains(generatedShortName)); - productShortNameMap.add(generatedShortName); - return generatedShortName; + } + throw new RuntimeException("Unable to generate unique short name"); } }