add ADBC tests for Go, C#, Java, Kotlin, Python, Clojure

Author: refset

.github/workflows/test.yml

@@ -12,110 +12,141 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
 
-    services:
-      xtdb:
-        image: ghcr.io/xtdb/xtdb:edge
-        ports:
-          - 5432:5432
-          - 8080:8080
-        options: >-
-          --health-cmd "curl --silent --fail http://localhost:8080/healthz/alive || exit 1"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    container:
-      image: archlinux:base
-      options: --user root
-
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
 
-      - name: Install Essential System Dependencies
-        run: |
-          chmod +x $GITHUB_WORKSPACE/.devcontainer/install-system-deps.sh
-          $GITHUB_WORKSPACE/.devcontainer/install-system-deps.sh essential
-
-      - name: Create Test User
-        run: |
-          useradd -m -s /bin/bash -u 1001 runner
-          echo "runner ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
-
-      - name: Install and configure mise
-        run: |
-          cp $GITHUB_WORKSPACE/.devcontainer/setup-mise.sh /tmp/setup-mise.sh
-          chmod +x /tmp/setup-mise.sh
-          su - runner -c "/tmp/setup-mise.sh runner $GITHUB_WORKSPACE"
-          rm /tmp/setup-mise.sh
-
-      - name: Install Language Runtimes
-        run: |
-          chown -R runner:runner $GITHUB_WORKSPACE
-          su - runner -c "cd $GITHUB_WORKSPACE && ~/.local/bin/mise install"
-
-      - name: Install Language-Specific Tools
-        run: |
-          su - runner -c "cd $GITHUB_WORKSPACE && for dir in python csharp babashka; do
-            if [ -d \"\$dir\" ] && [ -f \"\$dir/.mise.toml\" ]; then
-              echo \"Installing mise tools for \$dir...\"
-              (cd \"\$dir\" && ~/.local/bin/mise install) || echo \"Warning: Failed to install tools for \$dir\"
-            fi
-          done"
-
-      - name: Setup Language Runtime Packages
-        run: |
-          # Install system packages for Ruby, Elixir, PHP
-          $GITHUB_WORKSPACE/.devcontainer/install-system-deps.sh languages
-
-      - name: Setup Language Tools (Clojure, Elixir, PHP)
-        run: |
-          cp $GITHUB_WORKSPACE/.devcontainer/setup-language-tools.sh /tmp/setup-language-tools.sh
-          chmod +x /tmp/setup-language-tools.sh
-          su - runner -c "/tmp/setup-language-tools.sh all"
-          rm /tmp/setup-language-tools.sh
-
-      - name: Install Language Dependencies
-        run: |
-          su - runner -c "cd $GITHUB_WORKSPACE && \
-            eval \"\$(~/.local/bin/mise activate bash)\" && \
-            for dir in python node ruby go elixir php; do
-              if [ -d \"\$dir\" ]; then
-                echo \"Installing dependencies for \$dir...\"
-                (cd \"\$dir\" && ~/.local/bin/mise run deps 2>&1) || echo \"Warning: Failed to install deps for \$dir\"
-              fi
-            done"
-
-      - name: Wait for XTDB to be Ready
+      - name: Start XTDB with Flight SQL
         run: |
+          # Create XTDB config with Flight SQL enabled
+          # Write directly to avoid mount issues
+          cat > /tmp/xtdb.yaml << 'EOF'
+          server:
+            host: '*'
+            port: 5432
+
+          log: !Local
+            path: "/var/lib/xtdb/log"
+
+          storage: !Local
+            path: "/var/lib/xtdb/buffers"
+
+          healthz:
+            host: '*'
+            port: 8080
+
+          flightSql:
+            host: '*'
+            port: 9833
+          EOF
+
+          # Remove leading whitespace from heredoc (caused by YAML indentation)
+          sed -i 's/^          //' /tmp/xtdb.yaml
+
+          # Verify config
+          echo "XTDB config:"
+          cat /tmp/xtdb.yaml
+
+          # Start XTDB container with custom config that enables Flight SQL
+          docker run -d --name xtdb \
+            -p 5432:5432 -p 8080:8080 -p 9833:9833 \
+            -v "/tmp/xtdb.yaml:/config/xtdb.yaml:ro" \
+            ghcr.io/xtdb/xtdb:edge \
+            -f /config/xtdb.yaml
+
+          # Wait for XTDB to be ready
           for i in {1..60}; do
-            if curl --silent --fail http://xtdb:8080/healthz/alive; then
+            if curl --silent --fail http://localhost:8080/healthz/alive 2>/dev/null; then
               echo "XTDB is ready"
-              exit 0
+              break
             fi
+            echo "Waiting for XTDB... ($i/60)"
             sleep 2
           done
-          echo "XTDB did not start in time" && exit 1
 
-      - name: Verify XTDB Connection
-        run: |
-          $GITHUB_WORKSPACE/.devcontainer/install-system-deps.sh postgres
-          su - runner -c "psql -h xtdb -U xtdb -d xtdb -c 'SELECT 1'" || echo "PostgreSQL connection test failed"
-        env:
-          PGPASSWORD: xtdb
-
-      - name: Run All Tests
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Verify it started
+          if ! curl --silent --fail http://localhost:8080/healthz/alive; then
+            echo "XTDB did not start in time"
+            docker logs xtdb
+            exit 1
+          fi
+
+      - name: Pull Archlinux container
+        run: docker pull archlinux:base
+
+      - name: Run tests in Archlinux container
         run: |
-          su - runner -c "cd $GITHUB_WORKSPACE && \
-            export XTDB_HOST=xtdb && \
-            export XTDB_PORT=5432 && \
-            export XTDB_USER=xtdb && \
-            export XTDB_PASSWORD=xtdb && \
-            export XTDB_DATABASE=xtdb && \
-            eval \"\$(~/.local/bin/mise activate bash)\" && \
-            ~/.local/bin/mise run test:all"
+          docker run --rm \
+            --network host \
+            -v ${{ github.workspace }}:/workspace \
+            -w /workspace \
+            -e XTDB_HOST=localhost \
+            -e XTDB_PORT=5432 \
+            -e XTDB_USER=xtdb \
+            -e XTDB_PASSWORD=xtdb \
+            -e XTDB_DATABASE=xtdb \
+            -e GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \
+            archlinux:base \
+            bash -c '
+              set -e
+
+              # Install essential deps
+              chmod +x /workspace/.devcontainer/install-system-deps.sh
+              /workspace/.devcontainer/install-system-deps.sh essential
+
+              # Create test user
+              useradd -m -s /bin/bash -u 1001 runner
+              echo "runner ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+              # Install mise
+              cp /workspace/.devcontainer/setup-mise.sh /tmp/setup-mise.sh
+              chmod +x /tmp/setup-mise.sh
+              su - runner -c "/tmp/setup-mise.sh runner /workspace"
+
+              # Install language runtimes
+              chown -R runner:runner /workspace
+              su - runner -c "cd /workspace && ~/.local/bin/mise install"
+
+              # Install language-specific tools
+              su - runner -c "cd /workspace && for dir in python csharp babashka; do
+                if [ -d \"\$dir\" ] && [ -f \"\$dir/.mise.toml\" ]; then
+                  echo \"Installing mise tools for \$dir...\"
+                  (cd \"\$dir\" && ~/.local/bin/mise install) || echo \"Warning: Failed to install tools for \$dir\"
+                fi
+              done"
+
+              # Install system packages for Ruby, Elixir, PHP
+              /workspace/.devcontainer/install-system-deps.sh languages
+
+              # Setup Clojure, Elixir, PHP
+              cp /workspace/.devcontainer/setup-language-tools.sh /tmp/setup-language-tools.sh
+              chmod +x /tmp/setup-language-tools.sh
+              su - runner -c "/tmp/setup-language-tools.sh all"
+
+              # Install language dependencies
+              su - runner -c "cd /workspace && \
+                eval \"\$(~/.local/bin/mise activate bash)\" && \
+                for dir in python node ruby go elixir php; do
+                  if [ -d \"\$dir\" ]; then
+                    echo \"Installing dependencies for \$dir...\"
+                    (cd \"\$dir\" && ~/.local/bin/mise run deps 2>&1) || echo \"Warning: Failed to install deps for \$dir\"
+                  fi
+                done"
+
+              # Verify XTDB connection
+              /workspace/.devcontainer/install-system-deps.sh postgres
+              PGPASSWORD=xtdb su - runner -c "psql -h localhost -U xtdb -d xtdb -c \"SELECT 1\"" || echo "PostgreSQL connection test failed"
+
+              # Run all tests
+              su - runner -c "cd /workspace && \
+                export XTDB_HOST=localhost && \
+                export XTDB_PORT=5432 && \
+                export XTDB_USER=xtdb && \
+                export XTDB_PASSWORD=xtdb && \
+                export XTDB_DATABASE=xtdb && \
+                eval \"\$(~/.local/bin/mise activate bash)\" && \
+                ~/.local/bin/mise run test:all"
+            '
 
       - name: Upload Test Results
         if: always()
@@ -126,3 +157,7 @@ jobs:
             **/test-results/
             **/coverage/
           retention-days: 7
+
+      - name: Show XTDB logs on failure
+        if: failure()
+        run: docker logs xtdb

README.md

@@ -36,22 +36,24 @@ Feel free to hack on the examples - your Codespace is your own to explore!
 
 ## Language Feature Compatibility Matrix
 
-The following matrix shows features that are not supported in some languages:
-
-| Language | Transit-JSON (COPY) | Transit-Msgpack (COPY) | Transit-JSON (Parameters) |
-|----------|---------------------|------------------------|---------------------------|
-| **Python** | ✅ | ✅ | ✅ |
-| **Node.js** | ❌ | ❌ | ✅ |
-| **Go** | ✅ | ✅ | ✅ |
-| **Ruby** | ✅ | ✅ | ✅ |
-| **Java** | ✅ | ✅ | ✅ |
-| **Kotlin** | ✅ | ✅ | ✅ |
-| **C** | ✅ | ✅ | ✅ |
-| **C#** | ❌ | ❌ | ❌ |
-| **Clojure** | ✅ | ✅ | ✅ |
-| **Elixir** | ❌ | ❌ | ✅ |
-| **Babashka** | ❌ | ❌ | ❌ |
-| **PHP** | ❌ | ❌ | ✅ |
+The following matrix shows features that are not yet tested as working in some languages (i.e. no confirmed support, PRs welcome):
+
+| Language | Transit-JSON (COPY) | Transit-Msgpack (COPY) | Transit-JSON (Params) | ADBC |
+|----------|---------------------|------------------------|-----------------------|------|
+| **Python** | ✅ | ✅ | ✅ | ✅ |
+| **Node.js** | ❌ | ❌ | ✅ | ❌ |
+| **Go** | ✅ | ✅ | ✅ | ✅ |
+| **Ruby** | ✅ | ✅ | ✅ | ❌ |
+| **Java** | ✅ | ✅ | ✅ | ✅ |
+| **Kotlin** | ✅ | ✅ | ✅ | ✅ |
+| **C** | ✅ | ✅ | ✅ | ❌ |
+| **C#** | ❌ | ❌ | ❌ | ✅ |
+| **Clojure** | ✅ | ✅ | ✅ | ❌ |
+| **Elixir** | ❌ | ❌ | ✅ | ❌ |
+| **Babashka** | ❌ | ❌ | ❌ | ❌ |
+| **PHP** | ❌ | ❌ | ✅ | ❌ |
+
+[Apache Arrow ADBC](https://arrow.apache.org/adbc/) drivers connect via gRPC (port 9833) using Arrow Flight SQL for high-performance columnar data transfer. In-process ADBC usage on the JVM is also supported.
 
 ### Updating the Matrix
 
@@ -61,7 +63,7 @@ The compatibility matrix is automatically generated from test output. To regener
 python3 scripts/generate-feature-matrix.py
 ```
 
-Languages report only **unsupported** features. All features are assumed supported by default, minimizing maintenance burden.
+Languages report only **unsupported** features. All features are assumed supported by default.
 
 ### Testing
 

babashka/bb/my_project/main.clj

@@ -3,7 +3,7 @@
 
 (defn -main [& _args]
   (def db {:dbtype   "postgresql"
-           :host     "xtdb"
+           :host     (or (System/getenv "XTDB_HOST") "xtdb")
            :dbname   "xtdb"
            :port     5432})
 

babashka/test/xtdb/xtdb_test.clj

@@ -4,7 +4,7 @@
             [cheshire.core :as json]))
 
 (def db-config {:dbtype "postgresql"
-                :host "xtdb"
+                :host (or (System/getenv "XTDB_HOST") "xtdb")
                 :dbname "xtdb"
                 :port 5432})
 

c/trades.c

@@ -14,7 +14,10 @@
 /* Configuration */
 #define MAX_INT_STR_LEN 32
 #define MAX_QUERY_LEN 1024
-#define DEFAULT_DB_PARAMS "host=localhost port=5432 dbname=xtdb"
+/* Note: Default uses XTDB_HOST env var if set, otherwise "xtdb" */
+#define DEFAULT_DB_HOST "xtdb"
+#define DEFAULT_DB_PORT "5432"
+#define DEFAULT_DB_NAME "xtdb"
 #define LOG_BUF_SIZE 2048
 
 /* Exit codes */
@@ -567,8 +570,11 @@ int main(int argc, char *argv[])
 
     if (pos == 0)
     {
-        strncpy(connection_string, DEFAULT_DB_PARAMS, sizeof(connection_string) - 1);
-        connection_string[sizeof(connection_string) - 1] = '\0';
+        /* Use XTDB_HOST env var if set, otherwise default to "xtdb" */
+        const char *env_host = getenv("XTDB_HOST");
+        const char *default_host = (env_host && *env_host) ? env_host : DEFAULT_DB_HOST;
+        snprintf(connection_string, sizeof(connection_string),
+                 "host=%s port=%s dbname=%s", default_host, DEFAULT_DB_PORT, DEFAULT_DB_NAME);
     }
 
     if (!connect_db(connection_string))

c/xtdb_test.c

@@ -791,7 +791,15 @@ TEST(transit_json_copy_from) {
 int main(void) {
     srand(time(NULL));
 
-    PGconn *conn = PQconnectdb("host=xtdb port=5432 dbname=xtdb user=xtdb password=");
+    const char *host = getenv("XTDB_HOST");
+    if (!host || *host == '\0') {
+        host = "xtdb";
+    }
+
+    char conn_str[256];
+    snprintf(conn_str, sizeof(conn_str), "host=%s port=5432 dbname=xtdb user=xtdb password=", host);
+
+    PGconn *conn = PQconnectdb(conn_str);
 
     if (PQstatus(conn) != CONNECTION_OK) {
         fprintf(stderr, "Connection failed: %s\n", PQerrorMessage(conn));

clojure/.mise.toml

@@ -32,5 +32,5 @@ run = "clojure -M:dev:repl"
 description = "Run Clojure tests"
 run = """
 mise run setup
-clojure -M:test 2>&1 | grep -v 'Downloading:'
+clojure -J--add-opens=java.base/java.nio=ALL-UNNAMED -M:test 2>&1 | grep -v 'Downloading:'
 """

clojure/deps.edn

@@ -12,6 +12,10 @@
          ;; JSON processing
          org.clojure/data.json {:mvn/version "2.5.0"}
 
+         ;; Arrow ADBC Flight SQL (same versions as XTDB)
+         org.apache.arrow.adbc/adbc-driver-flight-sql {:mvn/version "0.20.0"}
+         org.apache.arrow/arrow-memory-netty {:mvn/version "18.3.0"}
+
          ;; helpful for debugging the loading data script
          org.clojure/tools.logging {:mvn/version "1.2.4"}
          org.slf4j/slf4j-simple {:mvn/version "2.0.9"}}

clojure/dev/user.clj

@@ -1,8 +1,10 @@
 (ns user
   (:require [xtdb.api :as xt]))
 
+(def xtdb-host (or (System/getenv "XTDB_HOST") "xtdb"))
+
 (defn get-client []
-  (xt/client {:host "xtdb"
+  (xt/client {:host xtdb-host
               :port 5432
               :user "xtdb"}))
 
@@ -82,7 +84,7 @@
 
   ;; To explore the Sakila dataset using datafy/nav tooling with next.jdbc
   (defn get-jdbc-conn []
-    (jdbc/get-connection "jdbc:xtdb://xtdb:5432/xtdb?options=-c%20TimeZone=UTC"))
+    (jdbc/get-connection (str "jdbc:xtdb://" xtdb-host ":5432/xtdb?options=-c%20TimeZone=UTC")))
 
   (tap> (jdbc/execute! (get-jdbc-conn) ["select * from inventory"]
                        {:schema-opts {:pk "_id"}}))