Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions .github/workflows/macos-cmake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ on:
required: true
description: "The ref we want to compile"
type: string
secrets:
BUILD_CACHE_KEY:
required: true

permissions:
contents: read
Expand All @@ -25,18 +28,19 @@ jobs:
os: [ macos-14, macos-15 ]
shard: [ BqDriver ]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
ref: ${{ inputs.checkout-ref }}
- uses: google-github-actions/auth@v2
persist-credentials: false
- uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.13
with:
create_credentials_file: true
credentials_json: ${{ secrets.BUILD_CACHE_KEY }}
- uses: actions/setup-python@v5
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
id: py311
with:
python-version: '3.11'
- uses: google-github-actions/setup-gcloud@v2
- uses: google-github-actions/setup-gcloud@e427ad8a34f8676edf47cf7d7925499adf3eb74f # v2.2.1
with:
version: '500.0.0'
env:
Expand All @@ -59,34 +63,39 @@ jobs:
- name: cmake version
run: cmake --version


- name: Install bison flex for 'arrow'
run: brew install bison flex
- name: Download and Install vcpkg
working-directory: "${{runner.temp}}"
env:
VCPKG_VERSION: ${{ steps.dynamic.outputs.vcpkg-version }}
run: |
git clone --branch ${{ steps.dynamic.outputs.vcpkg-version }} https://github.com/microsoft/vcpkg.git
git clone --branch "$VCPKG_VERSION" https://github.com/microsoft/vcpkg.git
vcpkg/bootstrap-vcpkg.sh -disableMetrics
- name: Install iODBC
run: |
brew install libiodbc

- name: Setup the driver
env:
CLOUDSDK_PYTHON: ${{ steps.py311.outputs.python-path }}
run: |
export VCPKG_ROOT="${{ runner.temp }}/vcpkg"
export PATH="$VCPKG_ROOT:/opt/homebrew/bin:$PATH"
export LDFLAGS="-L$(brew --prefix libiodbc)/lib -liconv"
export PATH="$(dirname ${{ steps.py311.outputs.python-path }}):$PATH"
export CLOUDSDK_PYTHON="${{ steps.py311.outputs.python-path }}"
export PATH="$(dirname "$CLOUDSDK_PYTHON"):$PATH"
bash ci/dependencies/driver-manager-setup-osx.sh

- name: Build cpp-bigquery-odbc
env:
CLOUDSDK_PYTHON: ${{ steps.py311.outputs.python-path }}
run: |
export ODBC_PREFIX="$(brew --prefix libiodbc)"
export ODBC_INCLUDE_PATH="$ODBC_PREFIX/include"
export VCPKG_ROOT="${{ runner.temp }}/vcpkg"

export PATH="$(dirname ${{ steps.py311.outputs.python-path }}):$(brew --prefix)/bin:/opt/homebrew/bin:$VCPKG_ROOT:$PATH"
export CLOUDSDK_PYTHON="${{ steps.py311.outputs.python-path }}"
export PATH="$(dirname "$CLOUDSDK_PYTHON"):$(brew --prefix)/bin:/opt/homebrew/bin:$VCPKG_ROOT:$PATH"
export LDFLAGS="-L$(brew --prefix libiodbc)/lib -liconv"
bash ci/gha/builds/macos-cmake.sh
env:
Expand Down
36 changes: 20 additions & 16 deletions .github/workflows/macos-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,21 +24,22 @@ jobs:
os: [ macos-14]
shard: [ BqDriver ]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
fetch-depth: 0
persist-credentials: false

- uses: google-github-actions/auth@v2
- uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.13
with:
create_credentials_file: true
credentials_json: ${{ secrets.BUILD_CACHE_KEY }}

- uses: actions/setup-python@v5
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
id: py311
with:
python-version: '3.11'

- uses: google-github-actions/setup-gcloud@v2
- uses: google-github-actions/setup-gcloud@e427ad8a34f8676edf47cf7d7925499adf3eb74f # v2.2.1
with:
version: '500.0.0'
env:
Expand All @@ -65,30 +66,29 @@ jobs:
- name: cmake version
run: cmake --version



- name: Install bison flex for 'arrow'
run: brew install bison flex

- name: Download and Install vcpkg
working-directory: "${{runner.temp}}"
env:
VCPKG_VERSION: ${{ steps.dynamic.outputs.vcpkg-version }}
run: |
git clone --branch ${{ steps.dynamic.outputs.vcpkg-version }} https://github.com/microsoft/vcpkg.git
git clone --branch "$VCPKG_VERSION" https://github.com/microsoft/vcpkg.git
vcpkg/bootstrap-vcpkg.sh -disableMetrics

- name: Install iODBC
run: |
brew install libiodbc



- name: Setup and Build cpp-bigquery-odbc driver
env:
CLOUDSDK_PYTHON: ${{ steps.py311.outputs.python-path }}
run: |
export VCPKG_ROOT="${{ runner.temp }}/vcpkg"
export PATH="$VCPKG_ROOT:/opt/homebrew/bin:$PATH"
export LDFLAGS="-L$(brew --prefix libiodbc)/lib -liconv"
export PATH="$(dirname ${{ steps.py311.outputs.python-path }}):$PATH"
export CLOUDSDK_PYTHON="${{ steps.py311.outputs.python-path }}"
export PATH="$(dirname "$CLOUDSDK_PYTHON"):$PATH"
bash ci/gha/builds/macos-release-setup.sh
export ODBC_INCLUDE_PATH="/opt/homebrew/opt/libiodbc/include"
bash ci/gha/builds/macos-cmake.sh
Expand All @@ -112,6 +112,8 @@ jobs:
echo "version=$VERSION" >> "$GITHUB_OUTPUT"

- name: Create TAR Package
env:
DRIVER_VERSION: ${{ steps.version.outputs.version }}
run: |
set -euo pipefail

Expand Down Expand Up @@ -139,20 +141,22 @@ jobs:
cp -rv ci/etc/roots.pem release_package/lib/roots.pem

# Create TAR.GZ file
TAR_NAME="ODBCDriverforBigQuery_macos_${{ steps.version.outputs.version }}.tar.gz"
TAR_NAME="ODBCDriverforBigQuery_macos_${DRIVER_VERSION}.tar.gz"
tar -czvf "$TAR_NAME" -C release_package .

echo "TAR package created: $TAR_NAME"

- name: Attest build provenance
uses: actions/attest-build-provenance@v2
uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2.4.0
with:
subject-path: ODBCDriverforBigQuery_macos_${{ steps.version.outputs.version }}.tar.gz

- name: Upload TAR to GitHub Release
uses: softprops/action-gh-release@v2
with:
files: ODBCDriverforBigQuery_macos_${{ steps.version.outputs.version }}.tar.gz
env:
DRIVER_VERSION: ${{ steps.version.outputs.version }}
BRANCH_NAME: ${{ github.ref_name }}
run: |
gh release upload "$BRANCH_NAME" "ODBCDriverforBigQuery_macos_${DRIVER_VERSION}.tar.gz" --clobber

env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down
34 changes: 23 additions & 11 deletions .github/workflows/test-runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || '' }}
cancel-in-progress: true

permissions:
contents: read

jobs:
pre-flight:
# Manual runs (workflow_dispatch) are treated as internal/trusted
Expand Down Expand Up @@ -75,7 +78,8 @@ jobs:
uses: ./.github/workflows/macos-cmake.yml
with:
checkout-ref: ${{ needs.pre-flight.outputs.checkout-sha }}
secrets: inherit
secrets:
BUILD_CACHE_KEY: ${{ secrets.BUILD_CACHE_KEY }}
windows-cmake:
name: Windows-CMake
if: |
Expand All @@ -84,7 +88,8 @@ jobs:
uses: ./.github/workflows/windows-cmake.yml
with:
checkout-ref: ${{ needs.pre-flight.outputs.checkout-sha }}
secrets: inherit
secrets:
BUILD_CACHE_KEY: ${{ secrets.BUILD_CACHE_KEY }}

windows-benchmark-bq:
name: Windows-Benchmark (Google Driver)
Expand All @@ -98,7 +103,8 @@ jobs:
with:
checkout-ref: ${{ needs.pre-flight.outputs.checkout-sha }}
build_shard: 'BqDriver'
secrets: inherit
secrets:
BUILD_CACHE_KEY: ${{ secrets.BUILD_CACHE_KEY }}

windows-benchmark-existing:
name: Windows-Benchmark (Existing Driver)
Expand All @@ -112,7 +118,8 @@ jobs:
with:
checkout-ref: ${{ needs.pre-flight.outputs.checkout-sha }}
build_shard: 'Core'
secrets: inherit
secrets:
BUILD_CACHE_KEY: ${{ secrets.BUILD_CACHE_KEY }}

windows-benchmark-results:
name: Windows-Benchmark (Generate Results Table)
Expand All @@ -125,24 +132,27 @@ jobs:
permissions:
contents: read
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ needs.pre-flight.outputs.checkout-sha }}
persist-credentials: false

- uses: google-github-actions/auth@v2
- uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.13
with:
create_credentials_file: true
credentials_json: ${{ secrets.BUILD_CACHE_KEY }}

- uses: google-github-actions/setup-gcloud@v2
- uses: google-github-actions/setup-gcloud@e427ad8a34f8676edf47cf7d7925499adf3eb74f # v2.2.1

- name: Download Results from GCS
env:
BRANCH_NAME: ${{ github.ref_name }}
run: |
mkdir -p benchmark_results

gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/current_bq.txt || true
gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/current_bq.txt || true

gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/performance_benchmark_results_Core.txt ./benchmark_results/current_core.txt || true
gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/performance_benchmark_results_Core.txt ./benchmark_results/current_core.txt || true

gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/main_bq.txt || true

Expand Down Expand Up @@ -258,6 +268,8 @@ jobs:
EOF

- name: Upload Table to GCS
env:
BRANCH_NAME: ${{ github.ref_name }}
run: |
gcloud storage cp benchmark_summary_table.txt gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/
echo "Uploaded benchmark table to gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/benchmark_summary_table.txt"
gcloud storage cp benchmark_summary_table.txt gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/
echo "Uploaded benchmark table to gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/benchmark_summary_table.txt"
22 changes: 15 additions & 7 deletions .github/workflows/windows-benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ on:
description: "The driver shard to test (Core or BqDriver)"
type: string
default: "BqDriver"
secrets:
BUILD_CACHE_KEY:
required: true
workflow_dispatch:
inputs:
build_shard:
Expand All @@ -34,37 +37,40 @@ jobs:
DRIVER_ARCH: x64
BUILD_SHARD: ${{ inputs.build_shard }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.checkout-ref || github.ref }}
persist-credentials: false

- uses: google-github-actions/auth@v2
- uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.13
with:
create_credentials_file: true
credentials_json: ${{ secrets.BUILD_CACHE_KEY }}

- uses: actions/setup-python@v5
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
id: py311
with:
python-version: '3.11'

- uses: google-github-actions/setup-gcloud@v2
- uses: google-github-actions/setup-gcloud@e427ad8a34f8676edf47cf7d7925499adf3eb74f # v2.2.1
env:
CLOUDSDK_PYTHON: ${{ steps.py311.outputs.python-path }}

- name: Install CMake
uses: lukka/get-cmake@latest
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
with:
cmakeVersion: "3.29.0"


- name: Download Installer from GCS
if: env.BUILD_SHARD == 'BqDriver'
shell: bash
env:
BRANCH_NAME: ${{ github.ref_name }}
run: |
mkdir -p installer_dl
# Sanitize the branch name replacing any character not in [a-zA-Z0-9-] with '_'
SANITIZED_BRANCH=$(echo "${{ github.ref_name }}" | sed -E 's/[^a-zA-Z0-9-]/_/g')
SANITIZED_BRANCH=$(echo "$BRANCH_NAME" | sed -E 's/[^a-zA-Z0-9-]/_/g')
gcloud storage cp gs://odbc-integration-builds/${SANITIZED_BRANCH}/*.msi ./installer_dl/

- name: Install the ODBC Driver
Expand Down Expand Up @@ -119,6 +125,8 @@ jobs:

- name: Run Benchmarks & Save Output
shell: bash
env:
BRANCH_NAME: ${{ github.ref_name }}
run: |
export RESULTS_FILE="performance_benchmark_results_${BUILD_SHARD}.txt"
export GOOGLE_APPLICATION_CREDENTIALS="C:\\b\\service_account_auth_keys.json"
Expand All @@ -141,7 +149,7 @@ jobs:
set -e

echo "Uploading results to GCS..."
gcloud storage cp "$RESULTS_FILE" gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/
gcloud storage cp "$RESULTS_FILE" gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/

if [ $TEST_EXIT_CODE -ne 0 ]; then
echo "ERROR: Benchmark executable failed with exit code $TEST_EXIT_CODE."
Expand Down
Loading
Loading