diff --git a/.github/workflows/s3_test.yml b/.github/workflows/s3_test.yml new file mode 100644 index 000000000..74ab11025 --- /dev/null +++ b/.github/workflows/s3_test.yml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# S3-backed tests against MinIO (Linux and macOS only). +name: S3 Tests + +on: + push: + branches: + - '**' + - '!dependabot/**' + tags: + - '**' + pull_request: + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +env: + ICEBERG_HOME: /tmp/iceberg + +jobs: + s3-minio: + name: S3 (${{ matrix.title }}) + runs-on: ${{ matrix.runs-on }} + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + include: + - title: AMD64 Ubuntu 24.04 + runs-on: ubuntu-24.04 + CC: gcc-14 + CXX: g++-14 + - title: AArch64 macOS 26 + runs-on: macos-26 + env: + ICEBERG_TEST_S3_URI: s3://iceberg-test + AWS_ACCESS_KEY_ID: minio + AWS_SECRET_ACCESS_KEY: minio123 + AWS_DEFAULT_REGION: us-east-1 + AWS_ENDPOINT_URL: http://127.0.0.1:9000 + AWS_EC2_METADATA_DISABLED: "TRUE" + steps: + - name: Checkout iceberg-cpp + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - name: Install dependencies on Ubuntu + if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} + shell: bash + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + - name: Set Ubuntu Compilers + if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} + run: | + echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV + echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV + - name: Start MinIO + shell: bash + run: bash ci/scripts/start_minio.sh + - name: Build and test Iceberg with S3 + shell: bash + run: ci/scripts/build_iceberg.sh "$(pwd)" OFF OFF ON diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f6ac1143..dffabe8ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,13 +43,6 @@ jobs: timeout-minutes: 30 strategy: fail-fast: false - env: - ICEBERG_TEST_S3_URI: s3://iceberg-test - AWS_ACCESS_KEY_ID: minio - AWS_SECRET_ACCESS_KEY: minio123 - AWS_DEFAULT_REGION: us-east-1 - AWS_ENDPOINT_URL: http://127.0.0.1:9000 - AWS_EC2_METADATA_DISABLED: "TRUE" steps: - name: Checkout iceberg-cpp uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -58,9 +51,6 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - - name: Start MinIO - shell: bash - run: bash ci/scripts/start_minio.sh - name: Build Iceberg shell: bash env: @@ -79,21 +69,11 @@ jobs: timeout-minutes: 30 strategy: fail-fast: false - env: - ICEBERG_TEST_S3_URI: s3://iceberg-test - AWS_ACCESS_KEY_ID: minio - AWS_SECRET_ACCESS_KEY: minio123 - AWS_DEFAULT_REGION: us-east-1 - AWS_ENDPOINT_URL: http://127.0.0.1:9000 - AWS_EC2_METADATA_DISABLED: "TRUE" steps: - name: Checkout iceberg-cpp uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false - - name: Start MinIO - shell: bash - run: bash ci/scripts/start_minio.sh - name: Build Iceberg shell: bash run: ci/scripts/build_iceberg.sh $(pwd) @@ -106,13 +86,6 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - env: - ICEBERG_TEST_S3_URI: s3://iceberg-test - AWS_ACCESS_KEY_ID: minio - AWS_SECRET_ACCESS_KEY: minio123 - AWS_DEFAULT_REGION: us-east-1 - AWS_ENDPOINT_URL: http://127.0.0.1:9000 - AWS_EC2_METADATA_DISABLED: "TRUE" steps: - name: Checkout iceberg-cpp uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -128,9 +101,6 @@ jobs: vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows - name: Setup sccache uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - name: Start MinIO - shell: bash - run: bash ci/scripts/start_minio.sh - name: Build Iceberg shell: pwsh env: diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index 5d69dd074..4c88427e3 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -16,6 +16,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Usage: build_iceberg.sh [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF] set -eux @@ -23,6 +25,7 @@ source_dir=${1} build_dir=${1}/build build_rest_integration_test=${2:-OFF} build_enable_sccache=${3:-OFF} +build_enable_s3=${4:-OFF} mkdir ${build_dir} pushd ${build_dir} @@ -36,10 +39,15 @@ CMAKE_ARGS=( "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}" "-DICEBERG_BUILD_STATIC=ON" "-DICEBERG_BUILD_SHARED=ON" - "-DICEBERG_S3=ON" "-DICEBERG_BUILD_REST_INTEGRATION_TESTS=${build_rest_integration_test}" ) +if [[ "${build_enable_s3}" == "ON" ]]; then + CMAKE_ARGS+=("-DICEBERG_S3=ON") +else + CMAKE_ARGS+=("-DICEBERG_S3=OFF") +fi + if is_windows; then CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")