Add new extension yagp hooks collector

.github/workflows/build-cloudberry-rocky8.yml

@@ -320,6 +320,10 @@ jobs:
                                "gpcontrib/gp_sparse_vector:installcheck",
                                "gpcontrib/gp_toolkit:installcheck"]
               },
+              {"test":"gpcontrib-gp-stats-collector",
+               "make_configs":["gpcontrib/gp_stats_collector:installcheck"],
+               "extension":"gp_stats_collector"
+              },
               {"test":"ic-fixme",
                "make_configs":["src/test/regress:installcheck-fixme"],
                "enable_core_check":false
@@ -540,10 +544,11 @@ jobs:
         if: needs.check-skip.outputs.should_skip != 'true'
         env:
           SRC_DIR: ${{ github.workspace }}
+          CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector
         run: |
           set -eo pipefail
           chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh
-          if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
+          if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
             echo "::error::Configure script failed"
             exit 1
           fi
@@ -1400,6 +1405,7 @@ jobs:
         if: success() && needs.check-skip.outputs.should_skip != 'true'
         env:
           SRC_DIR: ${{ github.workspace }}
+          BUILD_DESTINATION: /usr/local/cloudberry-db
         shell: bash {0}
         run: |
           set -o pipefail
@@ -1423,6 +1429,30 @@ jobs:
           # 2. Follow the same pattern as optimizer
           # 3. Update matrix entries to include the new setting
 
+          # Create extension if required
+          if [[ "${{ matrix.extension != '' }}" == "true" ]]; then
+            case "${{ matrix.extension }}" in
+              gp_stats_collector)
+                if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \
+                  source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \
+                  gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \
+                  gpstop -ra && \
+                  echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \
+                        SHOW shared_preload_libraries; \
+                        TABLE pg_extension;' | \
+                    psql postgres"
+                then
+                    echo "Error creating gp_stats_collector extension"
+                    exit 1
+                fi
+                ;;
+              *)
+                echo "Unknown extension: ${{ matrix.extension }}"
+                exit 1
+                ;;
+            esac
+          fi
+
           # Set PostgreSQL options if defined
           PG_OPTS=""
           if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then

.github/workflows/build-cloudberry.yml

@@ -271,6 +271,10 @@ jobs:
                 },
                 "enable_core_check":false
               },
+              {"test":"gpcontrib-gp-stats-collector",
+               "make_configs":["gpcontrib/gp_stats_collector:installcheck"],
+               "extension":"gp_stats_collector"
+              },
               {"test":"ic-expandshrink",
                "make_configs":["src/test/isolation2:installcheck-expandshrink"]
               },
@@ -535,10 +539,11 @@ jobs:
         if: needs.check-skip.outputs.should_skip != 'true'
         env:
           SRC_DIR: ${{ github.workspace }}
+          CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector
         run: |
           set -eo pipefail
           chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh
-          if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
+          if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
             echo "::error::Configure script failed"
             exit 1
           fi
@@ -1403,6 +1408,7 @@ jobs:
         if: success() && needs.check-skip.outputs.should_skip != 'true'
         env:
           SRC_DIR: ${{ github.workspace }}
+          BUILD_DESTINATION: /usr/local/cloudberry-db
         shell: bash {0}
         run: |
           set -o pipefail
@@ -1432,6 +1438,30 @@ jobs:
             PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}"
           fi
 
+          # Create extension if required
+          if [[ "${{ matrix.extension != '' }}" == "true" ]]; then
+            case "${{ matrix.extension }}" in
+              gp_stats_collector)
+                if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \
+                  source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \
+                  gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \
+                  gpstop -ra && \
+                  echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \
+                        SHOW shared_preload_libraries; \
+                        TABLE pg_extension;' | \
+                    psql postgres"
+                then
+                    echo "Error creating gp_stats_collector extension"
+                    exit 1
+                fi
+                ;;
+              *)
+                echo "Unknown extension: ${{ matrix.extension }}"
+                exit 1
+                ;;
+            esac
+          fi
+
           if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then
             PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}"
           fi

.github/workflows/build-deb-cloudberry.yml

@@ -252,6 +252,10 @@ jobs:
                                "gpcontrib/gp_sparse_vector:installcheck",
                                "gpcontrib/gp_toolkit:installcheck"]
               },
+              {"test":"gpcontrib-gp-stats-collector",
+               "make_configs":["gpcontrib/gp_stats_collector:installcheck"],
+               "extension":"gp_stats_collector"
+              },
               {"test":"ic-cbdb-parallel",
                "make_configs":["src/test/regress:installcheck-cbdb-parallel"]
               }
@@ -448,13 +452,14 @@ jobs:
         shell: bash
         env:
           SRC_DIR: ${{ github.workspace }}
+          CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector
         run: |
           set -eo pipefail
 
           export BUILD_DESTINATION=${SRC_DIR}/debian/build
 
           chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh
-          if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
+          if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
             echo "::error::Configure script failed"
             exit 1
           fi
@@ -1341,6 +1346,7 @@ jobs:
         if: success() && needs.check-skip.outputs.should_skip != 'true'
         env:
           SRC_DIR: ${{ github.workspace }}
+          BUILD_DESTINATION: ${{ github.workspace }}/debian/build
         shell: bash {0}
         run: |
           set -o pipefail
@@ -1365,6 +1371,30 @@ jobs:
           # 3. Update matrix entries to include the new setting
 
 
+          # Create extension if required
+          if [[ "${{ matrix.extension != '' }}" == "true" ]]; then
+            case "${{ matrix.extension }}" in
+              gp_stats_collector)
+                if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \
+                  source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \
+                  gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \
+                  gpstop -ra && \
+                  echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \
+                        SHOW shared_preload_libraries; \
+                        TABLE pg_extension;' | \
+                    psql postgres"
+                then
+                    echo "Error creating gp_stats_collector extension"
+                    exit 1
+                fi
+                ;;
+              *)
+                echo "Unknown extension: ${{ matrix.extension }}"
+                exit 1
+                ;;
+            esac
+          fi
+
           # Set PostgreSQL options if defined
           PG_OPTS=""
           if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then

.gitignore

@@ -73,4 +73,4 @@ lib*.pc
 /compile_commands.json
 /tmp_install/
 /.cache/
-/install/
+/install/

configure

@@ -722,6 +722,7 @@ with_apr_config
 with_libcurl
 with_rt
 with_zstd
+with_gp_stats_collector
 with_libbz2
 LZ4_LIBS
 LZ4_CFLAGS
@@ -942,6 +943,7 @@ with_zlib
 with_lz4
 with_libbz2
 with_zstd
+with_gp_stats_collector
 with_rt
 with_libcurl
 with_apr_config
@@ -11150,6 +11152,32 @@ $as_echo "yes" >&6; }
 fi
 fi
 
+#
+# gp_stats_collector
+#
+
+
+
+# Check whether --with-gp-stats-collector was given.
+if test "${with_gp_stats_collector+set}" = set; then :
+  withval=$with_gp_stats_collector;
+  case $withval in
+    yes)
+      :
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-gp-stats-collector option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_gp_stats_collector=no
+
+fi
+
 #
 # Realtime library
 #

configure.ac

@@ -1365,6 +1365,24 @@ PGAC_ARG_BOOL(with, zstd, yes, [do not build with Zstandard],
 AC_MSG_RESULT([$with_zstd])
 AC_SUBST(with_zstd)
 
+#
+# gp_stats_collector
+#
+PGAC_ARG_BOOL(with, gp_stats_collector, no,
+              [build with stats collector extension])
+AC_SUBST(with_gp_stats_collector)
+
+if test "$with_gp_stats_collector" = yes; then
+  PKG_CHECK_MODULES([PROTOBUF], [protobuf >= 3.0.0],
+    [],
+    [AC_MSG_ERROR([protobuf >= 3.0.0 is required for gp_stats_collector])]
+  )
+  AC_PATH_PROG([PROTOC], [protoc], [no])
+  if test "$PROTOC" = no; then
+    AC_MSG_ERROR([protoc is required for gp_stats_collector but was not found in PATH])
+  fi
+fi
+
 if test "$with_zstd" = yes; then
   dnl zstd_errors.h was renamed from error_public.h in v1.4.0
   PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.4.0])

devops/build/automation/cloudberry/scripts/configure-cloudberry.sh

@@ -53,6 +53,7 @@
 #
 # Optional Environment Variables:
 #   LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs)
+#   CONFIGURE_EXTRA_OPTS - Args to pass to configure command
 #   ENABLE_DEBUG - Enable debug build options (true/false, defaults to
 #                  false)
 #
@@ -177,7 +178,8 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \
             --with-uuid=e2fs \
             ${CONFIGURE_MDBLOCALES_OPTS} \
             --with-includes=/usr/local/xerces-c/include \
-            --with-libraries=${BUILD_DESTINATION}/lib || exit 4
+            --with-libraries=${BUILD_DESTINATION}/lib \
+            ${CONFIGURE_EXTRA_OPTS:-""} || exit 4
 log_section_end "Configure"
 
 # Capture version information

gpcontrib/Makefile

@@ -35,6 +35,9 @@ else
                diskquota
 endif
 
+ifeq "$(with_gp_stats_collector)" "yes"
+	recurse_targets += gp_stats_collector
+endif
 ifeq "$(with_zstd)" "yes"
 	recurse_targets += zstd
 endif

gpcontrib/gp_stats_collector/.clang-format

@@ -0,0 +1,2 @@
+BasedOnStyle: LLVM
+SortIncludes: false

gpcontrib/gp_stats_collector/.gitignore

@@ -0,0 +1,5 @@
+*.o
+*.so
+src/protos/
+.vscode
+compile_commands.json

gpcontrib/gp_stats_collector/Makefile

@@ -0,0 +1,35 @@
+MODULE_big = gp_stats_collector
+EXTENSION  = gp_stats_collector
+DATA       = $(wildcard *--*.sql)
+REGRESS    = gpsc_cursors gpsc_dist gpsc_select gpsc_utf8_trim gpsc_utility gpsc_guc_cache gpsc_uds gpsc_locale
+
+PROTO_BASES = gpsc_plan gpsc_metrics gpsc_set_service
+PROTO_OBJS  = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES))
+
+C_OBJS   = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c))
+CPP_OBJS = $(patsubst %.cpp,%.o,$(wildcard src/*.cpp src/*/*.cpp))
+OBJS     = $(C_OBJS) $(CPP_OBJS) $(PROTO_OBJS)
+
+PG_CXXFLAGS += -Werror -Wall -Wno-unused-but-set-variable -std=c++17 -Isrc/protos -Isrc -Iinclude -DGPBUILD
+SHLIB_LINK += -lprotobuf -lstdc++
+EXTRA_CLEAN = src/protos
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = gpcontrib/gp_stats_collector
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
+
+src/protos/.done: $(wildcard protos/*.proto)
+	@mkdir -p src/protos
+	protoc -I /usr/include -I /usr/local/include -I . --cpp_out=src $^
+	for f in src/protos/*.pb.cc; do mv "$$f" "$${f%.cc}.cpp"; done
+	touch $@
+
+src/protos/%.pb.cpp src/protos/%.pb.h: src/protos/.done ;
+$(CPP_OBJS): src/protos/.done

gpcontrib/gp_stats_collector/README.md

@@ -0,0 +1,47 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+## GP Stats Collector
+
+An extension for collecting query execution metrics and reporting them to an external agent.
+
+### Collected Statistics
+
+#### 1. Query Lifecycle
+-   **What:** Captures query text, normalized query text, timestamps (submit, start, end, done), and user/database info.
+-   **GUC:** `gpsc.enable`.
+
+#### 2. `EXPLAIN` data
+-   **What:** Triggers generation of the `EXPLAIN (TEXT, COSTS, VERBOSE)` and captures it.
+-   **GUC:** `gpsc.enable`.
+
+#### 3. `EXPLAIN ANALYZE` data
+-   **What:** Triggers generation of the `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)` and captures it.
+-   **GUCs:** `gpsc.enable`, `gpsc.min_analyze_time`, `gpsc.enable_cdbstats`(ANALYZE), `gpsc.enable_analyze`(BUFFERS, TIMING, VERBOSE).
+
+#### 4. Other Metrics
+-   **What:** Captures Instrument, System, Network, Interconnect, Spill metrics.
+-   **GUC:** `gpsc.enable`.
+
+### General Configuration
+-   **Nested Queries:** When `gpsc.report_nested_queries` is `false`, only top-level queries are reported from the coordinator and segments, when `true`, both top-level and nested queries are reported from the coordinator, from segments collected as aggregates.
+-   **Data Destination:** All collected data is sent to a Unix Domain Socket. Configure the path with `gpsc.uds_path`.
+-   **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `gpsc.ignored_users_list`.
+-   **Trimming plans:** Query texts and execution plans are trimmed based on `gpsc.max_text_size` and `gpsc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB.
+-   **Analyze collection:** Analyze is sent if execution time exceeds `gpsc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `gpsc.enable_analyze` is true.