From d7a9780467593f15cc250f9725ca77efdbbeb758 Mon Sep 17 00:00:00 2001 From: ducndh Date: Fri, 6 Mar 2026 04:59:17 +0000 Subject: [PATCH 1/2] sirius result Marth 5th --- sirius/benchmark.sh | 51 ++++++++++++++---- sirius/dependencies.sh | 79 ++------------------------- sirius/results/lambda-GH200.json | 93 ++++++++++++++++---------------- 3 files changed, 91 insertions(+), 132 deletions(-) diff --git a/sirius/benchmark.sh b/sirius/benchmark.sh index 1c369f170..5518b008b 100755 --- a/sirius/benchmark.sh +++ b/sirius/benchmark.sh @@ -1,17 +1,47 @@ #!/bin/bash -# Install dependencies +# Install dependencies (ensure the filename matches your setup script) source dependencies.sh -# Build Sirius -git clone --recurse-submodules https://github.com/sirius-db/sirius.git +# Ensure Pixi is in the PATH for the current script +export PATH="$HOME/.pixi/bin:$PATH" + +# Clean up any broken previous runs +rm -rf sirius + +# Build Sirius from your optimized fork +git clone --recurse-submodules -b clickbench https://github.com/sirius-db/sirius.git cd sirius -git checkout clickbench -source setup_sirius.sh -make -j$(nproc) -export PATH="$PATH:`pwd`/build/release/" + +# Enable strict error checking: fail immediately if the build crashes +set -e + +# Set up the substrait extension as an external extension (required by sirius) +mkdir -p duckdb/extension_external +git clone https://github.com/duckdb/substrait.git duckdb/extension_external/substrait +git -C duckdb/extension_external/substrait reset --hard ec9f8725df7aa22bae7217ece2f221ac37563da4 + +# Install the pixi environment first so the prefix path exists +pixi install + +# LIBCUDF_ENV_PREFIX must be set before cmake runs (CMakeLists.txt checks it at configure time) +export LIBCUDF_ENV_PREFIX="$(pwd)/.pixi/envs/default" + +# Execute the build process inside the isolated Pixi environment +pixi run make -j$(nproc) + +# Inject the Pixi environment variables into the current bash session +eval "$(pixi shell-hook)" + +# Add the compiled duckdb binary to the PATH +export PATH="$(pwd)/build/release/:$PATH" + +# Return to the parent directory cd .. +# Turn strict error checking back off for the benchmark phase +set +e + # Load the data wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.parquet' @@ -19,15 +49,14 @@ echo -n "Load time: " command time -f '%e' duckdb hits.db -f create.sql -f load.sql # Run the queries - ./run.sh 2>&1 | tee log.txt echo -n "Data size: " wc -c hits.db -cat log.txt | - grep -P '^\d|Killed|Segmentation|^Run Time \(s\): real' | - sed -r -e 's/^.*(Killed|Segmentation).*$/null\nnull\nnull/; s/^Run Time \(s\): real\s*([0-9.]+).*$/\1/' | +cat log.txt | \ + grep -P '^\d|Killed|Segmentation|^Run Time \(s\): real' | \ + sed -r -e 's/^.*(Killed|Segmentation).*$/null\nnull\nnull/; s/^Run Time \(s\): real\s*([0-9.]+).*$/\1/' | \ awk '{ buf[i++] = $1 if (i == 4) { diff --git a/sirius/dependencies.sh b/sirius/dependencies.sh index eef2f1062..fff42de7d 100755 --- a/sirius/dependencies.sh +++ b/sirius/dependencies.sh @@ -1,80 +1,11 @@ #!/bin/bash echo "Checking dependencies..." -# DuckDB dependencies sudo apt-get update -y -sudo apt-get install -y git g++ ninja-build libssl-dev build-essential make ccache pip +sudo apt-get install -y git wget curl build-essential time -# CMake -CMAKE_REQUIRED_VERSION="3.30.4" -if command -v cmake &>/dev/null; then - CMAKE_INSTALLED_VERSION=$(cmake --version | head -n1 | awk '{print $3}') -else - CMAKE_INSTALLED_VERSION="0" -fi - -version_lt() { - [ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | head -n1)" != "$2" ] -} - -if version_lt "$CMAKE_INSTALLED_VERSION" "$CMAKE_REQUIRED_VERSION"; then - echo "CMake version too old, installing version $CMAKE_REQUIRED_VERSION..." - sudo apt purge --autoremove -y cmake - wget -qO- "https://github.com/Kitware/CMake/releases/download/v$CMAKE_REQUIRED_VERSION/cmake-$CMAKE_REQUIRED_VERSION.tar.gz" | tar -xz - cd "cmake-$CMAKE_REQUIRED_VERSION" - ./bootstrap - make -j$(nproc) - sudo make install - cd .. - rm -rf "cmake-$CMAKE_REQUIRED_VERSION" -fi - -# CONDA -MINICONDA_DIR="$HOME/miniconda3" -if [[ ! -d "$MINICONDA_DIR" ]]; then - echo "Miniconda not found. Installing..." - ARCH=$(uname -m) - if [[ "$ARCH" == "x86_64" ]]; then - MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" - elif [[ "$ARCH" == "aarch64" ]]; then - MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh" - else - echo "Unsupported architecture for Miniconda: $ARCH" - exit 1 - fi - wget -q $MINICONDA_URL -O /tmp/Miniconda3-latest.sh - bash /tmp/Miniconda3-latest.sh -b -p $HOME/miniconda3 - rm /tmp/Miniconda3-latest.sh -fi -source "$MINICONDA_DIR/etc/profile.d/conda.sh" - -# Accept ToS for non-interactive use -conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main -conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r - -# CUDF -CUDF_REQUIRED_VERSION="25.12" -CUDF_NIGHTLY=true -CONDA_CUDF_ENV_NAME="libcudf-env" -CONDA_BASE=$(conda info --base) -export LIBCUDF_ENV_PREFIX="$CONDA_BASE/envs/$CONDA_CUDF_ENV_NAME" - -check_libcudf_version() { - conda list --prefix "$1" 2>/dev/null | grep "^libcudf" | awk '{print $2}' -} -CUDF_INSTALLED_VERSION=$(check_libcudf_version "$LIBCUDF_ENV_PREFIX" || echo "") - -if [[ ! -d "$LIBCUDF_ENV_PREFIX" ]]; then - echo "Creating conda environment $CONDA_CUDF_ENV_NAME..." - conda create --name "$CONDA_CUDF_ENV_NAME" -y -fi -conda activate "$CONDA_CUDF_ENV_NAME" - -if [[ "$CUDF_INSTALLED_VERSION" != "$CUDF_REQUIRED_VERSION" ]]; then - echo "Installing/upgrading libcudf $CUDF_REQUIRED_VERSION..." - if [[ "$CUDF_NIGHTLY" = true ]]; then - conda install -c rapidsai -c conda-forge -c nvidia "rapidsai-nightly::libcudf=$CUDF_REQUIRED_VERSION" -y - else - conda install -c rapidsai -c conda-forge -c nvidia "rapidsai::libcudf=$CUDF_REQUIRED_VERSION" -y - fi +# Install Pixi globally so it is available to all scripts without PATH hacks +if ! command -v pixi &> /dev/null; then + echo "Pixi not found. Installing globally..." + curl -fsSL https://pixi.sh/install.sh | sudo PIXI_BIN_DIR=/usr/local/bin PIXI_NO_PATH_UPDATE=1 bash fi diff --git a/sirius/results/lambda-GH200.json b/sirius/results/lambda-GH200.json index d176f5265..b4e918bd0 100644 --- a/sirius/results/lambda-GH200.json +++ b/sirius/results/lambda-GH200.json @@ -1,58 +1,57 @@ { "system": "Sirius", - "date": "2025-11-07", + "date": "2026-03-06", "machine": "lambda-GH200", "cluster_size": 1, "proprietary": "no", "hardware": "gpu", "tuned": "no", "tags": ["C++","column-oriented","embedded"], - "load_time": 26.30, - "data_size": 26903326720, + "load_time": 21.54, + "data_size": 26887598080, "result": [ - [0.014,0.001,0.001], - [0.374,0.004,0.004], - [0.757,0.005,0.004], - [0.793,0.005,0.004], - [0.541,0.008,0.007], - [2.117,0.013,0.010], - [0.277,0.005,0.004], - [0.289,0.005,0.011], - [1.006,0.084,0.085], - [1.334,0.089,0.089], - [1.621,0.017,0.016], - [1.649,0.018,0.017], - [1.835,0.018,0.018], - [2.432,0.124,0.125], - [2.100,0.020,0.019], - [0.557,0.013,0.013], - [2.303,0.036,0.036], - [2.314,0.026,0.025], - [3.444,0.060,0.059], - [0.499,0.004,0.004], - [10.373,0.041,0.041], - [9.085,0.014,0.013], - [17.467,0.032,0.032], - [70.218,0.066,0.067], - [3.397,0.013,0.013], - [1.795,0.009,0.009], - [2.453,0.028,0.027], - [12.015,0.165,0.165], - [9.587,0.268,0.267], - [0.576,0.036,0.035], - [3.161,0.010,0.009], - [3.995,0.013,0.013], - [1.673,0.072,0.072], - [7.463,0.075,0.074], - [7.331,0.077,0.077], - [0.475,0.020,0.020], - [7.886,0.006,0.006], - [8.330,0.006,0.006], - [7.853,0.006,0.005], - [16.314,0.010,0.010], - [3.441,0.006,0.005], - [2.121,0.005,0.005], - [1.605,0.005,0.005] + [0.015,0.001,0.001], + [0.346,0.001,0.002], + [0.706,0.002,0.003], + [0.727,0.002,0.002], + [0.486,0.005,0.005], + [2.033,0.008,0.008], + [0.252,0.002,0.002], + [0.245,0.002,0.003], + [0.843,0.012,0.010], + [1.220,0.093,0.092], + [1.556,0.005,0.003], + [1.566,0.005,0.004], + [1.733,0.013,0.013], + [2.149,0.019,0.017], + [1.959,0.015,0.014], + [0.501,0.011,0.011], + [2.209,0.026,0.026], + [2.135,0.024,0.023], + [3.192,0.044,0.044], + [0.462,0.002,0.002], + [9.283,0.039,0.039], + [8.037,0.012,0.012], + [15.642,0.023,0.023], + [68.510,0.061,0.062], + [3.150,0.007,0.008], + [1.688,0.008,0.008], + [2.272,0.007,0.007], + [10.939,0.132,0.132], + [8.658,0.240,0.240], + [0.523,0.034,0.033], + [2.969,0.008,0.008], + [3.713,0.013,0.012], + [1.497,0.062,0.061], + [6.336,0.053,0.052], + [6.301,0.055,0.055], + [0.433,0.018,0.017], + [6.758,0.005,0.004], + [7.578,0.004,0.004], + [6.656,0.003,0.003], + [14.840,0.007,0.007], + [3.172,0.004,0.003], + [1.883,0.003,0.003], + [1.448,0.003,0.002] ] } - From 02562c74df2000342817d2e31bf208fdb8d676db Mon Sep 17 00:00:00 2001 From: ducndh Date: Tue, 10 Mar 2026 04:05:53 +0000 Subject: [PATCH 2/2] Change the script to use default branch and simplify install script. Add H100 result too --- sirius/benchmark.sh | 57 ++++++++++---------- sirius/dependencies.sh | 10 ++-- sirius/results/lambda-GH200.json | 90 +++++++++++++++---------------- sirius/results/p5.4xlarge.json | 91 ++++++++++++++++---------------- 4 files changed, 124 insertions(+), 124 deletions(-) diff --git a/sirius/benchmark.sh b/sirius/benchmark.sh index 5518b008b..c391439d8 100755 --- a/sirius/benchmark.sh +++ b/sirius/benchmark.sh @@ -1,66 +1,63 @@ #!/bin/bash +# ClickBench benchmark for Sirius (GPU-accelerated DuckDB extension) +# +# Usage: ./benchmark.sh +# Prerequisites: NVIDIA GPU with CUDA driver, internet access -# Install dependencies (ensure the filename matches your setup script) source dependencies.sh -# Ensure Pixi is in the PATH for the current script -export PATH="$HOME/.pixi/bin:$PATH" +# Verify pixi is available +if ! command -v pixi &> /dev/null; then + echo "Error: pixi not found. Check dependencies.sh output." + exit 1 +fi -# Clean up any broken previous runs +# --------------------------------------------------------------------------- +# 1. Build Sirius +# --------------------------------------------------------------------------- rm -rf sirius - -# Build Sirius from your optimized fork -git clone --recurse-submodules -b clickbench https://github.com/sirius-db/sirius.git +git clone --recurse-submodules https://github.com/sirius-db/sirius.git cd sirius -# Enable strict error checking: fail immediately if the build crashes set -e -# Set up the substrait extension as an external extension (required by sirius) -mkdir -p duckdb/extension_external -git clone https://github.com/duckdb/substrait.git duckdb/extension_external/substrait -git -C duckdb/extension_external/substrait reset --hard ec9f8725df7aa22bae7217ece2f221ac37563da4 - -# Install the pixi environment first so the prefix path exists pixi install - -# LIBCUDF_ENV_PREFIX must be set before cmake runs (CMakeLists.txt checks it at configure time) export LIBCUDF_ENV_PREFIX="$(pwd)/.pixi/envs/default" +pixi run make -j"$(nproc)" -# Execute the build process inside the isolated Pixi environment -pixi run make -j$(nproc) - -# Inject the Pixi environment variables into the current bash session +# Make the build artifacts available eval "$(pixi shell-hook)" - -# Add the compiled duckdb binary to the PATH -export PATH="$(pwd)/build/release/:$PATH" - -# Return to the parent directory +export PATH="$(pwd)/build/release:$PATH" cd .. -# Turn strict error checking back off for the benchmark phase set +e -# Load the data +# --------------------------------------------------------------------------- +# 2. Load data +# --------------------------------------------------------------------------- wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.parquet' echo -n "Load time: " command time -f '%e' duckdb hits.db -f create.sql -f load.sql -# Run the queries +# --------------------------------------------------------------------------- +# 3. Run benchmark +# --------------------------------------------------------------------------- ./run.sh 2>&1 | tee log.txt echo -n "Data size: " wc -c hits.db +# --------------------------------------------------------------------------- +# 4. Format results +# --------------------------------------------------------------------------- cat log.txt | \ grep -P '^\d|Killed|Segmentation|^Run Time \(s\): real' | \ - sed -r -e 's/^.*(Killed|Segmentation).*$/null\nnull\nnull/; s/^Run Time \(s\): real\s*([0-9.]+).*$/\1/' | \ + sed -r -e 's/^.(Killed|Segmentation).$/null\nnull\nnull/; s/^Run Time \(s\): real\s*([0-9.]+).*$/\1/' | \ awk '{ buf[i++] = $1 if (i == 4) { printf "[%s,%s,%s],\n", buf[1], buf[2], buf[3] i = 0 } - }' + }' \ No newline at end of file diff --git a/sirius/dependencies.sh b/sirius/dependencies.sh index fff42de7d..bf8c22513 100755 --- a/sirius/dependencies.sh +++ b/sirius/dependencies.sh @@ -1,11 +1,15 @@ #!/bin/bash -echo "Checking dependencies..." +#!/bin/bash +# Install system dependencies required to build Sirius and run ClickBench. +echo "Installing system dependencies..." sudo apt-get update -y sudo apt-get install -y git wget curl build-essential time -# Install Pixi globally so it is available to all scripts without PATH hacks +# Install Pixi package manager (used to manage CUDA/cuDF toolchain) if ! command -v pixi &> /dev/null; then - echo "Pixi not found. Installing globally..." + echo "Pixi not found. Installing..." curl -fsSL https://pixi.sh/install.sh | sudo PIXI_BIN_DIR=/usr/local/bin PIXI_NO_PATH_UPDATE=1 bash fi + +echo "All dependencies installed." diff --git a/sirius/results/lambda-GH200.json b/sirius/results/lambda-GH200.json index b4e918bd0..121680350 100644 --- a/sirius/results/lambda-GH200.json +++ b/sirius/results/lambda-GH200.json @@ -1,57 +1,57 @@ { "system": "Sirius", - "date": "2026-03-06", + "date": "2026-03-09", "machine": "lambda-GH200", "cluster_size": 1, "proprietary": "no", "hardware": "gpu", "tuned": "no", "tags": ["C++","column-oriented","embedded"], - "load_time": 21.54, + "load_time": 25.30, "data_size": 26887598080, "result": [ - [0.015,0.001,0.001], - [0.346,0.001,0.002], - [0.706,0.002,0.003], - [0.727,0.002,0.002], - [0.486,0.005,0.005], - [2.033,0.008,0.008], - [0.252,0.002,0.002], - [0.245,0.002,0.003], - [0.843,0.012,0.010], - [1.220,0.093,0.092], - [1.556,0.005,0.003], - [1.566,0.005,0.004], - [1.733,0.013,0.013], - [2.149,0.019,0.017], - [1.959,0.015,0.014], - [0.501,0.011,0.011], - [2.209,0.026,0.026], - [2.135,0.024,0.023], - [3.192,0.044,0.044], - [0.462,0.002,0.002], - [9.283,0.039,0.039], - [8.037,0.012,0.012], - [15.642,0.023,0.023], - [68.510,0.061,0.062], - [3.150,0.007,0.008], - [1.688,0.008,0.008], - [2.272,0.007,0.007], - [10.939,0.132,0.132], - [8.658,0.240,0.240], - [0.523,0.034,0.033], - [2.969,0.008,0.008], - [3.713,0.013,0.012], - [1.497,0.062,0.061], - [6.336,0.053,0.052], - [6.301,0.055,0.055], - [0.433,0.018,0.017], - [6.758,0.005,0.004], - [7.578,0.004,0.004], - [6.656,0.003,0.003], - [14.840,0.007,0.007], - [3.172,0.004,0.003], - [1.883,0.003,0.003], - [1.448,0.003,0.002] + [0.015,0.001,0.000], + [0.369,0.001,0.002], + [0.705,0.003,0.002], + [0.763,0.002,0.002], + [0.471,0.010,0.009], + [1.743,0.020,0.019], + [0.263,0.002,0.001], + [0.240,0.003,0.002], + [0.849,0.017,0.016], + [1.234,0.134,0.135], + [1.406,0.005,0.004], + [1.405,0.005,0.005], + [1.341,0.016,0.016], + [1.787,0.024,0.023], + [1.582,0.017,0.017], + [0.485,0.014,0.013], + [1.771,0.031,0.031], + [1.800,0.028,0.028], + [2.875,0.048,0.048], + [0.438,0.002,0.001], + [10.325,0.039,0.038], + [7.736,0.042,0.042], + [16.230,0.088,0.086], + [68.376,0.061,0.060], + [2.300,0.008,0.007], + [1.315,0.009,0.007], + [1.868,0.007,0.007], + [8.857,0.157,0.157], + [7.839,0.286,0.285], + [0.451,0.033,0.031], + [2.516,0.013,0.012], + [3.305,0.021,0.020], + [1.483,0.117,0.117], + [6.224,0.079,0.078], + [6.292,0.084,0.083], + [0.428,0.024,0.024], + [6.667,0.005,0.005], + [8.194,0.005,0.003], + [6.585,0.003,0.003], + [13.916,0.008,0.007], + [2.962,0.003,0.003], + [1.740,0.003,0.003], + [1.359,0.003,0.003] ] } diff --git a/sirius/results/p5.4xlarge.json b/sirius/results/p5.4xlarge.json index f3a5e40b2..2c2c94d3a 100644 --- a/sirius/results/p5.4xlarge.json +++ b/sirius/results/p5.4xlarge.json @@ -1,6 +1,6 @@ { "system": "Sirius", - "date": "2025-12-07", + "date": "2026-03-09", "machine": "p5.4xlarge", "cluster_size": 1, "proprietary": "no", @@ -10,49 +10,48 @@ "load_time": 237.18, "data_size": 26921938944, "result": [ - [0.028,0.000,0.001], - [1.468,0.002,0.001], - [3.838,0.003,0.002], - [3.447,0.002,0.002], - [3.028,0.005,0.005], - [6.696,0.008,0.008], - [1.168,0.002,0.002], - [1.840,0.003,0.002], - [5.249,0.092,0.093], - [7.568,0.098,0.097], - [6.409,0.015,0.015], - [6.295,0.015,0.015], - [6.824,0.069,0.068], - [9.907,0.191,0.192], - [8.017,0.075,0.074], - [3.551,0.012,0.011], - [9.769,0.121,0.121], - [9.751,0.024,0.024], - [13.273,0.161,0.160], - [2.906,0.002,0.001], - [41.637,0.040,0.040], - [47.953,0.012,0.012], - [81.308,0.031,0.031], - [224.830,0.065,0.065], - [11.813,0.008,0.008], - [6.547,0.007,0.007], - [12.030,0.128,0.129], - [43.496,0.164,0.163], - [32.878,0.274,0.274], - [2.226,0.038,0.036], - [13.673,0.008,0.007], - [19.490,0.012,0.012], - [13.002,0.081,0.081], - [42.224,0.545,0.547], - [42.345,0.556,0.552], - [2.619,0.019,0.019], - [45.643,0.009,0.009], - [33.959,0.004,0.004], - [45.765,0.003,0.003], - [73.482,0.020,0.020], - [16.912,0.003,0.003], - [14.177,0.003,0.002], - [9.339,0.003,0.002] + [0.028,0.001,0.000], + [1.331,0.002,0.002], + [3.862,0.003,0.002], + [3.468,0.002,0.002], + [3.334,0.010,0.009], + [6.078,0.020,0.019], + [1.217,0.003,0.001], + [1.696,0.003,0.003], + [4.523,0.018,0.018], + [8.371,0.141,0.140], + [6.283,0.006,0.004], + [8.090,0.006,0.005], + [6.460,0.016,0.017], + [9.827,0.024,0.024], + [7.784,0.018,0.017], + [3.622,0.015,0.014], + [8.422,0.032,0.031], + [9.131,0.029,0.029], + [12.604,0.051,0.051], + [3.382,0.002,0.002], + [43.575,0.041,0.040], + [47.567,0.044,0.043], + [75.496,0.091,0.090], + [218.280,0.065,0.064], + [11.345,0.008,0.007], + [6.019,0.008,0.008], + [11.176,0.007,0.007], + [44.371,0.151,0.151], + [33.973,0.288,0.286], + [2.267,0.038,0.039], + [11.605,0.014,0.013], + [16.902,0.021,0.022], + [13.003,0.126,0.124], + [43.721,0.080,0.080], + [43.739,0.085,0.085], + [3.042,0.025,0.025], + [46.731,0.005,0.005], + [34.366,0.005,0.004], + [46.588,0.004,0.003], + [74.694,0.008,0.007], + [16.826,0.004,0.003], + [13.496,0.004,0.003], + [9.627,0.003,0.003] ] -} - +} \ No newline at end of file