Run JDBC tests in Testcontainers (MVP)

.github/workflows/pxf-ci.yml

@@ -198,6 +198,39 @@ jobs:
         path: /tmp/singlecluster-rocky9-image.tar
         retention-days: 1
 
+  build-pxf-cbdb-testcontainer-image:
+    name: Build PXF-CBDB Testcontainer Image (${{ matrix.distro }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - distro: ubuntu
+            base_image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest
+            image_tag: pxf/cbdb-testcontainer:1
+          - distro: rocky9
+            base_image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
+            image_tag: pxf/cbdb-testcontainer-rocky9:1
+    steps:
+      - name: Checkout PXF source
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Build pxf-cbdb testcontainer image
+        run: |
+          docker build \
+            --build-arg BASE_IMAGE=${{ matrix.base_image }} \
+            -t ${{ matrix.image_tag }} \
+            automation/src/main/resources/testcontainers/pxf-cbdb
+          docker save ${{ matrix.image_tag }} > /tmp/pxf-cbdb-testcontainer-${{ matrix.distro }}.tar
+
+      - name: Upload pxf-cbdb testcontainer image
+        uses: actions/upload-artifact@v4
+        with:
+          name: pxf-cbdb-testcontainer-image-${{ matrix.distro }}
+          path: /tmp/pxf-cbdb-testcontainer-${{ matrix.distro }}.tar
+          retention-days: 1
+
   # Stage 2: Parallel test jobs using matrix strategy
   pxf-test:
     name: Test PXF - ${{ matrix.test_group }}
@@ -541,10 +574,123 @@ jobs:
           exit 1
         fi
 
+
+  # Stage 2c: Testcontainers-based tests
+  pxf-testcontainer-test:
+    name: "TC Test - ${{ matrix.tc_group }} (${{ matrix.use_fdw == 'true' && 'fdw' || 'external-table' }}, ${{ matrix.distro }})"
+    needs: [build-pxf-cbdb-testcontainer-image]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        tc_group:
+          - 'pxf-jdbc'
+        use_fdw:
+          - 'false'
+          - 'true'
+        distro:
+          - 'ubuntu'
+          - 'rocky9'
+    steps:
+      - name: Checkout PXF source
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          submodules: true
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: ${{ env.JAVA_VERSION }}
+
+      - name: Download pxf-cbdb testcontainer image
+        uses: actions/download-artifact@v4
+        with:
+          name: pxf-cbdb-testcontainer-image-${{ matrix.distro }}
+          path: /tmp
+
+      - name: Load pxf-cbdb testcontainer image
+        run: |
+          docker load < /tmp/pxf-cbdb-testcontainer-${{ matrix.distro }}.tar
+
+      - name: Build PXF stage artifacts (no unit tests)
+        run: |
+          make -C server stage-notest
+
+      - name: Run Testcontainers tests - ${{ matrix.tc_group }} (${{ matrix.use_fdw == 'true' && 'fdw' || 'external-table' }}, ${{ matrix.distro }})
+        id: run_test
+        continue-on-error: true
+        timeout-minutes: 120
+        working-directory: automation
+        env:
+          PXF_HOME: ${{ github.workspace }}/server/build/stage
+        run: |
+          make test-tc TC_GROUP=${{ matrix.tc_group }} USE_FDW=${{ matrix.use_fdw }} DISTRO=${{ matrix.distro }}
+
+      - name: Collect artifacts and generate stats
+        if: always()
+        id: collect_artifacts
+        run: |
+          mkdir -p artifacts/logs
+          TC_GROUP="${{ matrix.tc_group }}"
+          TEST_MODE="${{ matrix.use_fdw == 'true' && 'fdw' || 'external-table' }}"
+          TEST_RESULT="${{ steps.run_test.outcome }}"
+
+          # Copy test artifacts
+          cp -r automation/automation_logs/* artifacts/ 2>/dev/null || true
+
+          TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
+          for xml in automation/target/surefire-reports/TEST-*.xml; do
+            if [ -f "$xml" ]; then
+              tests=$(grep -oP 'tests="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+              failures=$(grep -oP 'failures="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+              errors=$(grep -oP 'errors="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+              skipped=$(grep -oP 'skipped="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+              TOTAL=$((TOTAL + tests))
+              FAILED=$((FAILED + failures + errors))
+              SKIPPED=$((SKIPPED + skipped))
+            fi
+          done
+          PASSED=$((TOTAL - FAILED - SKIPPED))
+
+          cat > artifacts/test_stats.json <<EOF
+          {
+            "group": "$TC_GROUP:$TEST_MODE:${{ matrix.distro }}",
+            "result": "$TEST_RESULT",
+            "total": $TOTAL,
+            "passed": $PASSED,
+            "failed": $FAILED,
+            "skipped": $SKIPPED
+          }
+          EOF
+
+          echo "failed_count=$FAILED" >> $GITHUB_OUTPUT
+          echo "skipped_count=$SKIPPED" >> $GITHUB_OUTPUT
+          echo "Test stats for tc:$TC_GROUP ($TEST_MODE, ${{ matrix.distro }}): total=$TOTAL, passed=$PASSED, failed=$FAILED, skipped=$SKIPPED"
+
+      - name: Upload test artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-tc-${{ matrix.tc_group }}-${{ matrix.use_fdw == 'true' && 'fdw' || 'external-table' }}-${{ matrix.distro }}
+          path: artifacts/**
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - name: Check test result
+        if: always()
+        run: |
+          FAILED_COUNT="${{ steps.collect_artifacts.outputs.failed_count || 0 }}"
+          if [ "${{ steps.run_test.outcome }}" == "failure" ] || [ "$FAILED_COUNT" -gt 0 ]; then
+            echo "Testcontainer test ${{ matrix.tc_group }} (${{ matrix.test_mode }}, ${{ matrix.distro }}) failed (Failures: $FAILED_COUNT)"
+            exit 1
+          fi
+
   # Stage 3: Summary job
   test-summary:
     name: Test Summary
-    needs: [pxf-test, pxf-test-rocky9]
+    needs: [pxf-test, pxf-test-rocky9, pxf-testcontainer-test]
     if: always()
     runs-on: ubuntu-latest
     steps:

automation/Makefile

@@ -22,6 +22,8 @@ ifneq "$(GROUP)" ""
 	MAVEN_TEST_OPTS+= -Dgroups=$(GROUP)
 endif
 
+EXCLUDED_GROUPS ?=
+
 MAVEN_TEST_OPTS+= -Djava.awt.headless=true -DuseFDW=$(USE_FDW) -Duser.timezone=UTC
 
 ifneq "$(OFFLINE)" "true"
@@ -94,7 +96,11 @@ MVN=mvn
 all: test
 
 check-env:
-	@if [ -z "$(PXF_HOME)" ]; then echo 'ERROR: PXF_HOME must be set'; exit 1; fi
+	@if [ -z "$(PXF_HOME)" ]; then \
+		echo 'ERROR: PXF_HOME must be set'; \
+		echo 'Example: export PXF_HOME="$(abspath ../server/build/stage)"'; \
+		exit 1; \
+	fi
 
 symlink_pxf_jars: check-env
 	@if [ -d "$(PXF_HOME)/application" ]; then \
@@ -118,7 +124,7 @@ symlink_pxf_jars: check-env
 	fi
 
 test: check-env clean-logs symlink_pxf_jars sync_cloud_configs sync_jdbc_config pxf_regress
-	$(MVN) $(MAVEN_TEST_OPTS) ${MAVEN_DEBUG_OPTS} test
+	$(MVN) $(MAVEN_TEST_OPTS) ${MAVEN_DEBUG_OPTS} -DexcludedGroups=testcontainers$${EXCLUDED_GROUPS:+,$$EXCLUDED_GROUPS} test
 
 clean: clean-logs
 	$(MVN) $(MAVEN_TEST_OPTS) clean
@@ -245,6 +251,20 @@ else
 	@ls src/test/java/org/apache/cloudberry/pxf/automation/features/*/*Test.java | sed 's/.*\///g' | sed 's/\.java//g' | awk '{print "* ", $$1}'
 endif
 
+# Run Testcontainers-based tests.
+# Usage:
+#   make test-tc                          => run all testcontainers tests (Ubuntu)
+#   make test-tc TC_GROUP=pxf-jdbc        => run only pxf-jdbc group
+#   make test-tc DISTRO=rocky9            => run with Rocky Linux 9 base image
+.PHONY: test-tc
+test-tc: check-env symlink_pxf_jars pxf_regress
+	$(MVN) -B -e -Djava.awt.headless=true -Duser.timezone=UTC \
+		-DuseFDW=$(USE_FDW) \
+		-Dpxf.test.distro=$(or $(DISTRO),$(PXF_TEST_DISTRO),ubuntu) \
+		-Dgroups=$(or $(TC_GROUP),testcontainers) \
+		-DexcludedGroups= \
+		test
+
 .PHONY: pxf_regress
 pxf_regress:
 	$(MAKE) -C pxf_regress

automation/pom.xml

@@ -76,6 +76,12 @@
         </plugins>
 
         <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <includes>
+                    <include>**/*</include>
+                </includes>
+            </resource>
             <resource>
                 <directory>src/test/resources</directory>
                 <includes>
@@ -177,6 +183,30 @@
             <version>4.2.0</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <version>2.0.3</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.17.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.17.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.26.2</version>
+        </dependency>
+
         <dependency>
             <groupId>org.jsystemtest</groupId>
             <artifactId>jsystemCore</artifactId>
@@ -262,19 +292,19 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
-            <version>2.14.3</version>
+            <version>2.20.2</version>
         </dependency>
 
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.14.3</version>
+            <version>2.20.2</version>
         </dependency>
 
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
-            <version>2.14.3</version>
+            <version>2.20</version>
         </dependency>
 
         <dependency>