feat: Re-adapt cloudberry/pxf_fdw to Cloudberry (#31)

Co-authored-by: Nikolay Antonov <ostinru@gmail.com>

Author: MisterRaindrop

.github/workflows/pxf-ci.yml

@@ -103,6 +103,7 @@ jobs:
         test_group:
           - cli
           - external-table
+          - fdw
           - server
           - sanity
           - smoke
@@ -118,6 +119,7 @@ jobs:
           - s3
           - features
           - gpdb
+          - gpdb_fdw
           - load
           - pxf_extension
     steps:

Makefile

@@ -6,8 +6,8 @@ export PXF_MODULES
 PXF_VERSION ?= $(shell cat version)
 export PXF_VERSION
 
-FDW_SUPPORT = $(shell $(PG_CONFIG) --version | egrep "PostgreSQL 1[2-5]")
-FDW_SUPPORT =
+export SKIP_EXTERNAL_TABLE_BUILD_REASON
+export SKIP_FDW_BUILD_REASON
 
 SOURCE_EXTENSION_DIR = external-table
 TARGET_EXTENSION_DIR = gpextable
@@ -24,29 +24,31 @@ all: extensions cli server
 
 extensions: external-table fdw
 
-external-table cli server:
+external-table:
+ifeq ($(SKIP_EXTERNAL_TABLE_BUILD_REASON),)
 	@echo "===> Compiling [$@] module <==="
 	make -C $@
+else
+	@echo "Skipping building external-table extension because $(SKIP_EXTERNAL_TABLE_BUILD_REASON)"
+endif
 
-fdw:
-ifneq ($(FDW_SUPPORT),)
+ifeq ($(SKIP_FDW_BUILD_REASON),)
+	@echo "===> Compiling [$@] module <==="
 	make -C fdw
+else
+	@echo "Skipping building FDW extension because $(SKIP_FDW_BUILD_REASON)"
 endif
 
-cli:
-	make -C cli
-
-server:
-	make -C server
+cli server:
+	@echo "===> Compiling [$@] module <==="
+	make -C $@
 
 clean:
 	rm -rf build
 	make -C $(SOURCE_EXTENSION_DIR) clean-all
 	make -C cli clean
 	make -C server clean
-ifneq ($(FDW_SUPPORT),)
 	make -C fdw clean
-endif
 
 test:
 ifeq ($(SKIP_FDW_BUILD_REASON),)
@@ -61,18 +63,30 @@ it:
 	make -C automation TEST=$(TEST)
 
 install:
-	make -C $(SOURCE_EXTENSION_DIR) install
-	make -C cli install
-	make -C server install
-ifneq ($(FDW_SUPPORT),)
-	make -C fdw install
+ifneq ($(SKIP_EXTERNAL_TABLE_BUILD_REASON),)
+	@echo "Skipping installing FDW extension because $(SKIP_EXTERNAL_TABLE_BUILD_REASON)"
+	$(eval PXF_MODULES := $(filter-out external-table,$(PXF_MODULES)))
+endif
+ifneq ($(SKIP_FDW_BUILD_REASON),)
+	@echo "Skipping installing FDW extension because $(SKIP_FDW_BUILD_REASON)"
+	$(eval PXF_MODULES := $(filter-out fdw,$(PXF_MODULES)))
 endif
+	set -e ;\
+	for module in $${PXF_MODULES[@]}; do \
+		echo "===> Installing [$${module}] module <===" ;\
+		make -C $${module} install ;\
+	done ;\
+	echo "===> PXF installation is complete <==="
 
 install-server:
 	make -C server install-server
 
 stage:
 	rm -rf build/stage
+ifneq ($(SKIP_EXTERNAL_TABLE_PACKAGE_REASON),)
+	@echo "Skipping staging FDW extension because $(SKIP_EXTERNAL_TABLE_PACKAGE_REASON)"
+	$(eval PXF_MODULES := $(filter-out external-table,$(PXF_MODULES)))
+endif
 ifneq ($(SKIP_FDW_PACKAGE_REASON),)
 	@echo "Skipping staging FDW extension because $(SKIP_FDW_PACKAGE_REASON)"
 	$(eval PXF_MODULES := $(filter-out fdw,$(PXF_MODULES)))

automation/src/test/java/org/greenplum/pxf/automation/features/writable/HdfsWritableTextTest.java

@@ -1,5 +1,6 @@
 package org.greenplum.pxf.automation.features.writable;
 
+import annotations.FailsWithFDW;
 import annotations.SkipForFDW;
 import annotations.WorksWithFDW;
 import org.apache.commons.lang.StringUtils;
@@ -75,6 +76,7 @@ protected void beforeClass() throws Exception {
      *
      * @throws Exception if test fails to run
      */
+    @FailsWithFDW
     @Test(groups = {"features", "gpdb", "security"})
     public void textFormatInsertNoProfile() throws Exception {
 

ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberrry.sh

@@ -100,7 +100,7 @@ cd ~/workspace/cloudberry
             --enable-orafce \
             --enable-orca \
             --disable-pax \
-            --enable-pxf \
+            --disable-pxf \
             --enable-tap-tests \
             --with-gssapi \
             --with-ldap \

ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh

@@ -28,7 +28,7 @@ cd "${CLOUDBERRY_SRC}"
             --enable-orafce \
             --enable-orca \
             --disable-pax \
-            --enable-pxf \
+            --disable-pxf \
             --enable-tap-tests \
             --with-gssapi \
             --with-ldap \

ci/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh

@@ -549,6 +549,7 @@ feature_test(){
 }
 
 gpdb_test() {
+  local use_fdw="$1"
   export PROTOCOL=HDFS
   export PXF_HOME=${PXF_HOME:-/usr/local/pxf}
   export PATH="${PXF_HOME}/bin:${PATH}"
@@ -570,10 +571,20 @@ gpdb_test() {
   # Ensure PXF points to local HDFS/Hive/HBase configs
   configure_pxf_default_hdfs_server
 
-  echo "[run_tests] Starting GROUP=gpdb"
-  make GROUP="gpdb" || true
-  save_test_reports "gpdb"
-  echo "[run_tests] GROUP=gpdb finished"
+  local extra_args=""
+  if [[ "$use_fdw" == "true" ]]; then
+    extra_args="USE_FDW=true"
+  else
+    extra_args="USE_FDW=false"
+  fi
+  echo "[run_tests] Starting GROUP=gpdb $extra_args"
+  make GROUP="gpdb" $extra_args || true
+  if [[ "$use_fdw" == "true" ]]; then
+      save_test_reports "gpdb_fdw"
+  else
+      save_test_reports "gpdb"
+  fi
+  echo "[run_tests] GROUP=gpdb $extra_args finished"
 }
 
 pxf_extension_test(){
@@ -709,7 +720,7 @@ generate_test_summary() {
 
     local group=$(basename "$group_dir")
     # Skip if it's not a test group directory
-    [[ "$group" =~ ^(smoke|hcatalog|hcfs|hdfs|hive|gpdb|sanity|hbase|profile|jdbc|proxy|unused|s3|features|load|performance|pxfExtensionVersion2|pxfExtensionVersion2_1|pxfFdwExtensionVersion1|pxfFdwExtensionVersion2)$ ]] || continue
+    [[ "$group" =~ ^(smoke|hcatalog|hcfs|hdfs|hive|gpdb|sanity|hbase|profile|jdbc|proxy|unused|s3|features|load|performance|pxfExtensionVersion2|pxfExtensionVersion2_1|pxfFdwExtensionVersion1|pxfFdwExtensionVersion2|fdw|gpdb_fdw)$ ]] || continue
 
     echo "Processing $group test reports from $group_dir"
 
@@ -841,6 +852,13 @@ run_single_group() {
       cd "${REPO_ROOT}/external-table"
       make installcheck
       ;;
+    fdw)
+      cd "${REPO_ROOT}/fdw"
+      make test
+      ;;
+    gpdb_fdw)
+      gpdb_test "true"
+      ;;
     server)
       cd "${REPO_ROOT}/server"
       ./gradlew test
@@ -873,7 +891,7 @@ run_single_group() {
       feature_test
       ;;
     gpdb)
-      gpdb_test
+      gpdb_test "false"
       ;;
     pxf_extension)
       pxf_extension_test
@@ -898,7 +916,7 @@ run_single_group() {
       ;;
     *)
       echo "Unknown test group: $group"
-      echo "Available groups: cli, external-table, server, sanity, smoke, hdfs, hcatalog, hcfs, hive, hbase, profile, jdbc, proxy, unused, s3, features, gpdb, load, performance, bench, pxf_extension"
+      echo "Available groups: cli, external-table, fdw, server, sanity, smoke, hdfs, hcatalog, hcfs, hive, hbase, profile, jdbc, proxy, unused, s3, features, gpdb, gpdb_fdw, load, performance, bench, pxf_extension"
       exit 1
       ;;
   esac

fdw/README.md

@@ -1,6 +1,6 @@
-# PXF Foreign Data Wrapper for Greenplum and PostgreSQL
+# PXF Foreign Data Wrapper for Cloudberry
 
-This Greenplum extension implements a Foreign Data Wrapper (FDW) for PXF.
+This Cloudberry extension implements a Foreign Data Wrapper (FDW) for PXF.
 
 PXF is a query federation engine that accesses data residing in external systems
 such as Hadoop, Hive, HBase, relational databases, S3, Google Cloud Storage,
@@ -10,9 +10,9 @@ among other external systems.
 
 ## Compile
 
-To compile the PXF foreign data wrapper, we need a Greenplum 6+ installation and libcurl.
+To compile the PXF foreign data wrapper, we need a Cloudberry installation and libcurl.
 
-    export PATH=/usr/local/greenplum-db/bin/:$PATH
+    export PATH=/usr/local/cloudberry-db/bin/:$PATH
 
     make
 

fdw/libchurl.c

@@ -16,19 +16,15 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-#if PG_VERSION_NUM >= 90600
 #include "postgres.h"
-#endif
-
 #include "libchurl.h"
-#if PG_VERSION_NUM >= 90600
 #include "lib/stringinfo.h"
-#endif
 #include "miscadmin.h"
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/guc.h"
-#include "utils/jsonapi.h"
+#include "common/jsonapi.h"
+#include "mb/pg_wchar.h"
 
 /* include libcurl without typecheck.
  * This allows wrapping curl_easy_setopt to be wrapped
@@ -98,13 +94,6 @@ typedef struct churl_settings
 	struct curl_slist *headers;
 } churl_settings;
 
-/* the null action object used for pure validation */
-static JsonSemAction nullSemAction =
-{
-	NULL, NULL, NULL, NULL, NULL,
-	NULL, NULL, NULL, NULL, NULL
-};
-
 churl_context *churl_new_context(void);
 static void		create_curl_handle(churl_context *context);
 static void		set_curl_option(churl_context *context, CURLoption option, const void *data);
@@ -864,13 +853,8 @@ fill_internal_buffer(churl_context *context, int want)
 
 			if (errno == EINTR || errno == EAGAIN)
 				continue;
-#if PG_VERSION_NUM >= 90600
 			elog(ERROR, "internal error: select failed on curl_multi_fdset (maxfd %d) (%d - %m)",
 				 maxfd, save_errno);
-#else
-			elog(ERROR, "internal error: select failed on curl_multi_fdset (maxfd %d) (%d - %s)",
-				 maxfd, save_errno, strerror(errno));
-#endif
 		}
 		multi_perform(context);
 	}
@@ -1056,7 +1040,7 @@ IsValidJson(text *json)
 	PG_TRY();
 	{
 		/* validate it */
-		lex = makeJsonLexContext(json, false);
+		lex = makeJsonLexContextCstringLen(text_to_cstring(json), VARSIZE_ANY_EXHDR(json), GetDatabaseEncoding(), false);
 		pg_parse_json(lex, &nullSemAction);
 	}
 	PG_CATCH();

fdw/pxf_bridge.c

@@ -27,11 +27,7 @@
 /* helper function declarations */
 static void BuildUriForRead(PxfFdwScanState *pxfsstate);
 static void BuildUriForWrite(PxfFdwModifyState *pxfmstate);
-#if PG_VERSION_NUM >= 90600
 static size_t FillBuffer(PxfFdwScanState *pxfsstate, char *start, int minlen, int maxlen);
-#else
-static size_t FillBuffer(PxfFdwScanState *pxfsstate, char *start, size_t size);
-#endif
 
 /*
  * Clean up churl related data structures from the PXF FDW modify state.
@@ -102,20 +98,12 @@ PxfBridgeExportStart(PxfFdwModifyState *pxfmstate)
  * Reads data from the PXF server into the given buffer of a given size
  */
 int
-#if PG_VERSION_NUM >= 90600
 PxfBridgeRead(void *outbuf, int minlen, int maxlen, void *extra)
-#else
-PxfBridgeRead(void *outbuf, int datasize, void *extra)
-#endif
 {
 	size_t		n = 0;
 	PxfFdwScanState *pxfsstate = (PxfFdwScanState *) extra;
 
-#if PG_VERSION_NUM >= 90600
 	n = FillBuffer(pxfsstate, outbuf, minlen, maxlen);
-#else
-	n = FillBuffer(pxfsstate, outbuf, datasize);
-#endif
 
 	if (n == 0)
 	{
@@ -176,28 +164,16 @@ BuildUriForWrite(PxfFdwModifyState *pxfmstate)
  * Read data from churl until the buffer is full or there is no more data to be read
  */
 static size_t
-#if PG_VERSION_NUM >= 90600
 FillBuffer(PxfFdwScanState *pxfsstate, char *start, int minlen, int maxlen)
-#else
-FillBuffer(PxfFdwScanState *pxfsstate, char *start, size_t size)
-#endif
 {
 	size_t		n = 0;
 	char	   *ptr = start;
-#if PG_VERSION_NUM >= 90600
 	char	   *minend = ptr + minlen;
 	char	   *maxend = ptr + maxlen;
 
 	while (ptr < minend)
 	{
 		n = churl_read(pxfsstate->churl_handle, ptr, maxend - ptr);
-#else
-	char	   *end = ptr + size;
-
-	while (ptr < end)
-	{
-		n = churl_read(pxfsstate->churl_handle, ptr, end - ptr);
-#endif
 		if (n == 0)
 			break;