From 623fb88c59aead7fabc84d6e174050018e166898 Mon Sep 17 00:00:00 2001 From: aram price Date: Mon, 13 Jul 2026 19:17:53 -0700 Subject: [PATCH] Add rust-1-linux as dependency of all ruby packages for YJIT/ZJIT support - Vendor rust-1-linux (1.97.0) from bosh-package-rust-release via spec.lock - Add rust-1-linux as a compile-time dependency of ruby-3.3, ruby-3.4, and ruby-4.0 - Drop Clang support: use explicit GCC/G++ on Linux; Rust is now in PATH during ./configure so Ruby auto-detects and enables YJIT (3.3/3.4) and ZJIT (4.0) - Patch common.mk: replace -C lto=thin with -C lto=no before ./configure because GNU ld (used with GCC) cannot link LLVM bitcode objects emitted by thin LTO; disabling thin LTO makes Rust emit plain native objects that GCC links cleanly - Add bump-rust-1-linux CI job: triggered by new bosh-package-rust-release releases, runs bosh vendor-package + tests all ruby versions in parallel before finalizing - Add rust-release git resource to pipeline (watches .final_builds for new releases) - CI: verify that RubyVM::YJIT is compiled in and can be enabled at runtime in all ruby test jobs ai-assisted=yes [TNZ-109821] Bosh ruby package compiles with rustc --- ci/pipeline.yml | 97 +++++++++++++++++++ ci/templates/jobs/ruby-test/templates/run.erb | 18 ++++ ci/templates/packages/ruby/packaging.erb | 24 +++-- ci/templates/packages/ruby/spec.erb | 3 + ci/templates/src/compile.env.erb | 11 --- config/blobs.yml | 3 + jobs/ruby-3.3-test/templates/run | 18 ++++ jobs/ruby-3.4-test/templates/run | 18 ++++ jobs/ruby-4.0-test/templates/run | 18 ++++ packages/ruby-3.3/packaging | 24 +++-- packages/ruby-3.3/spec | 3 + packages/ruby-3.4/packaging | 24 +++-- packages/ruby-3.4/spec | 3 + packages/ruby-4.0/packaging | 24 +++-- packages/ruby-4.0/spec | 3 + packages/rust-1-linux/spec.lock | 2 + src/compile-3.3.env | 11 --- src/compile-3.4.env | 11 --- src/compile-4.0.env | 11 --- 19 files changed, 246 insertions(+), 80 deletions(-) create mode 100644 packages/rust-1-linux/spec.lock diff --git a/ci/pipeline.yml b/ci/pipeline.yml index c84e4c4..677bd66 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -127,6 +127,95 @@ jobs: repository: release_repo tag: release_metadata/tag-name #@ end +- name: bump-rust-1-linux + serial: true + serial_groups: + - bump + plan: + - in_parallel: + - get: rust-release + trigger: true + - get: ruby-release + - get: stemcell + resource: ubuntu-noble + - get: bosh-src + - get: semver + params: + bump: minor + - get: bosh-shared-ci + - get: docker-cpi-image + - task: bump-rust-package + file: rust-release/ci/tasks/shared/bump-rust-package/task.yml + image: docker-cpi-image + input_mapping: + release-repo: ruby-release + params: + GIT_USER_NAME: CI Bot + GIT_USER_EMAIL: bots@cloudfoundry.org + PACKAGE: rust-1-linux + PRIVATE_YML: | + --- + blobstore: + provider: gcs + options: + credentials_source: static + json_key: '((gcp_json_key))' + - in_parallel: + - task: test-3.3 + file: ruby-release/ci/tasks/test.yml + image: docker-cpi-image + privileged: true + timeout: 2h + input_mapping: + ruby-release: release-repo + params: + RUBY_VERSION: "3.3" + STEMCELL: ubuntu-noble + - task: test-3.4 + file: ruby-release/ci/tasks/test.yml + image: docker-cpi-image + privileged: true + timeout: 2h + input_mapping: + ruby-release: release-repo + params: + RUBY_VERSION: "3.4" + STEMCELL: ubuntu-noble + - task: test-4.0 + file: ruby-release/ci/tasks/test.yml + image: docker-cpi-image + privileged: true + timeout: 2h + input_mapping: + ruby-release: release-repo + params: + RUBY_VERSION: "4.0" + STEMCELL: ubuntu-noble + - task: finalize + file: bosh-shared-ci/tasks/release/create-final-release.yml + input_mapping: + release_repo: release-repo + version: semver + params: + GIT_USER_NAME: CI Bot + GIT_USER_EMAIL: bots@cloudfoundry.org + PRIVATE_YML: | + --- + blobstore: + provider: gcs + options: + credentials_source: static + json_key: '((gcp_json_key))' + - in_parallel: + - put: semver + params: + file: semver/version + - put: ruby-release + params: + rebase: true + repository: release_repo + tag: release_metadata/tag-name + - name: build-docker-image plan: - in_parallel: @@ -177,6 +266,14 @@ resources: source: uri: https://github.com/cloudfoundry/bosh.git +- name: rust-release + type: git + source: + uri: https://github.com/cloudfoundry/bosh-package-rust-release.git + branch: main + paths: + - .final_builds/packages/rust-1-linux/index.yml + - name: ruby-release type: git source: diff --git a/ci/templates/jobs/ruby-test/templates/run.erb b/ci/templates/jobs/ruby-test/templates/run.erb index e9c1403..c74ca14 100644 --- a/ci/templates/jobs/ruby-test/templates/run.erb +++ b/ci/templates/jobs/ruby-test/templates/run.erb @@ -38,3 +38,21 @@ else echo "wrong rubygems version: $gems_version" exit 1 fi + +echo "Testing that Ruby was compiled with Rust (YJIT support)" + +if ruby -e 'exit(1) unless defined?(RubyVM::YJIT)'; then + echo "YJIT is compiled into Ruby" +else + echo "ERROR: YJIT is NOT compiled into Ruby. Was Rust available during compilation?" + exit 1 +fi + +echo "Testing that YJIT can be enabled at runtime" + +if ruby --yjit -e 'RubyVM::YJIT.enable; raise "YJIT not enabled" unless RubyVM::YJIT.enabled?; puts "YJIT enabled successfully"'; then + echo "YJIT is functional" +else + echo "ERROR: YJIT could not be enabled" + exit 1 +fi diff --git a/ci/templates/packages/ruby/packaging.erb b/ci/templates/packages/ruby/packaging.erb index bcb83ed..37335bb 100644 --- a/ci/templates/packages/ruby/packaging.erb +++ b/ci/templates/packages/ruby/packaging.erb @@ -12,17 +12,17 @@ if [ $(uname -s) == "Darwin" ]; then PATH="${brew_prefix_openssl}/bin:$PATH" with_openssl_dir="--with-openssl-dir=$brew_prefix_openssl" fi -fi +else + export CC="gcc" + export CXX="g++" -# Use Clang if available; the resulting Ruby is faster -if command -v clang &> /dev/null -then - export CC="$(command -v clang)" -fi + echo "GCC version:" + $CC --version + + source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/rust-1-linux/bosh/compile.env" -if command -v clang++ &> /dev/null -then - export CXX="$(command -v clang++)" + echo "Rust version:" + rustc --version --verbose fi openssl_version=$(openssl version) @@ -58,6 +58,12 @@ tar xzf "ruby-${RUBY_VERSION}.tar.gz" cp "${BOSH_COMPILE_TARGET}"/config/config.{guess,sub} ./tool fi + if [ "$(uname -s)" != "Darwin" ]; then + # GNU ld cannot handle LLVM bitcode objects produced by thin LTO; disable it + # so Rust emits plain native objects that GCC can link without issues. + sed -i 's/-C lto=thin/-C lto=no/g' common.mk + fi + LDFLAGS="-Wl,-rpath -Wl,${BOSH_INSTALL_TARGET}" ./configure --prefix="${BOSH_INSTALL_TARGET}" --disable-install-doc --with-opt-dir="${BOSH_INSTALL_TARGET}" --without-gmp $with_openssl_dir make make install diff --git a/ci/templates/packages/ruby/spec.erb b/ci/templates/packages/ruby/spec.erb index da6ca9f..1dc1a58 100644 --- a/ci/templates/packages/ruby/spec.erb +++ b/ci/templates/packages/ruby/spec.erb @@ -1,6 +1,9 @@ --- name: <%= ruby_package_name %> +dependencies: +- rust-1-linux + files: - compile-<%= ruby_version %>.env - config/config.guess diff --git a/ci/templates/src/compile.env.erb b/ci/templates/src/compile.env.erb index 30a3aca..beb8443 100644 --- a/ci/templates/src/compile.env.erb +++ b/ci/templates/src/compile.env.erb @@ -3,17 +3,6 @@ # shellcheck disable=1090 source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/${PACKAGE_NAME}/bosh/runtime.env" -# Use Clang if available; the resulting Ruby is faster -if command -v clang &> /dev/null; then - CC="$(command -v clang)" - export CC -fi - -if command -v clang++ &> /dev/null; then - CXX="$(command -v clang++)" - export CXX -fi - bosh_bundle() { bundle config set --local no_prune 'true' bundle config set --local without 'development test' diff --git a/config/blobs.yml b/config/blobs.yml index 498e50e..eab25e0 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -22,6 +22,9 @@ rubygems-4.0.16.tgz: size: 978912 object_id: 2a34dec1-5ae5-45eb-5f4d-bf98503ce743 sha: sha256:ea9c669526af82874f8f33f69bea1b6ddd99283756e598227a9a890035a5a06a +rust-1-linux/rust-1.97.0-x86_64-unknown-linux-gnu.tar.gz: + size: 384617996 + sha: sha256:eb89b20287153391c49ebcdb7fd91b683a12438d129bfb92eadcc495545af3a7 yaml-0.2.5.tar.gz: size: 609454 object_id: b5eeae2b-1496-4bc2-556f-efac50c8bcfa diff --git a/jobs/ruby-3.3-test/templates/run b/jobs/ruby-3.3-test/templates/run index cef1186..d2d6f7c 100644 --- a/jobs/ruby-3.3-test/templates/run +++ b/jobs/ruby-3.3-test/templates/run @@ -38,3 +38,21 @@ else echo "wrong rubygems version: $gems_version" exit 1 fi + +echo "Testing that Ruby was compiled with Rust (YJIT support)" + +if ruby -e 'exit(1) unless defined?(RubyVM::YJIT)'; then + echo "YJIT is compiled into Ruby" +else + echo "ERROR: YJIT is NOT compiled into Ruby. Was Rust available during compilation?" + exit 1 +fi + +echo "Testing that YJIT can be enabled at runtime" + +if ruby --yjit -e 'RubyVM::YJIT.enable; raise "YJIT not enabled" unless RubyVM::YJIT.enabled?; puts "YJIT enabled successfully"'; then + echo "YJIT is functional" +else + echo "ERROR: YJIT could not be enabled" + exit 1 +fi diff --git a/jobs/ruby-3.4-test/templates/run b/jobs/ruby-3.4-test/templates/run index 43ab9c7..d072b0d 100644 --- a/jobs/ruby-3.4-test/templates/run +++ b/jobs/ruby-3.4-test/templates/run @@ -38,3 +38,21 @@ else echo "wrong rubygems version: $gems_version" exit 1 fi + +echo "Testing that Ruby was compiled with Rust (YJIT support)" + +if ruby -e 'exit(1) unless defined?(RubyVM::YJIT)'; then + echo "YJIT is compiled into Ruby" +else + echo "ERROR: YJIT is NOT compiled into Ruby. Was Rust available during compilation?" + exit 1 +fi + +echo "Testing that YJIT can be enabled at runtime" + +if ruby --yjit -e 'RubyVM::YJIT.enable; raise "YJIT not enabled" unless RubyVM::YJIT.enabled?; puts "YJIT enabled successfully"'; then + echo "YJIT is functional" +else + echo "ERROR: YJIT could not be enabled" + exit 1 +fi diff --git a/jobs/ruby-4.0-test/templates/run b/jobs/ruby-4.0-test/templates/run index fa36936..a928de8 100644 --- a/jobs/ruby-4.0-test/templates/run +++ b/jobs/ruby-4.0-test/templates/run @@ -38,3 +38,21 @@ else echo "wrong rubygems version: $gems_version" exit 1 fi + +echo "Testing that Ruby was compiled with Rust (YJIT support)" + +if ruby -e 'exit(1) unless defined?(RubyVM::YJIT)'; then + echo "YJIT is compiled into Ruby" +else + echo "ERROR: YJIT is NOT compiled into Ruby. Was Rust available during compilation?" + exit 1 +fi + +echo "Testing that YJIT can be enabled at runtime" + +if ruby --yjit -e 'RubyVM::YJIT.enable; raise "YJIT not enabled" unless RubyVM::YJIT.enabled?; puts "YJIT enabled successfully"'; then + echo "YJIT is functional" +else + echo "ERROR: YJIT could not be enabled" + exit 1 +fi diff --git a/packages/ruby-3.3/packaging b/packages/ruby-3.3/packaging index 3e583eb..9b933b1 100644 --- a/packages/ruby-3.3/packaging +++ b/packages/ruby-3.3/packaging @@ -12,17 +12,17 @@ if [ $(uname -s) == "Darwin" ]; then PATH="${brew_prefix_openssl}/bin:$PATH" with_openssl_dir="--with-openssl-dir=$brew_prefix_openssl" fi -fi +else + export CC="gcc" + export CXX="g++" -# Use Clang if available; the resulting Ruby is faster -if command -v clang &> /dev/null -then - export CC="$(command -v clang)" -fi + echo "GCC version:" + $CC --version + + source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/rust-1-linux/bosh/compile.env" -if command -v clang++ &> /dev/null -then - export CXX="$(command -v clang++)" + echo "Rust version:" + rustc --version --verbose fi openssl_version=$(openssl version) @@ -58,6 +58,12 @@ tar xzf "ruby-${RUBY_VERSION}.tar.gz" cp "${BOSH_COMPILE_TARGET}"/config/config.{guess,sub} ./tool fi + if [ "$(uname -s)" != "Darwin" ]; then + # GNU ld cannot handle LLVM bitcode objects produced by thin LTO; disable it + # so Rust emits plain native objects that GCC can link without issues. + sed -i 's/-C lto=thin/-C lto=no/g' common.mk + fi + LDFLAGS="-Wl,-rpath -Wl,${BOSH_INSTALL_TARGET}" ./configure --prefix="${BOSH_INSTALL_TARGET}" --disable-install-doc --with-opt-dir="${BOSH_INSTALL_TARGET}" --without-gmp $with_openssl_dir make make install diff --git a/packages/ruby-3.3/spec b/packages/ruby-3.3/spec index ab909db..f56fbab 100644 --- a/packages/ruby-3.3/spec +++ b/packages/ruby-3.3/spec @@ -1,6 +1,9 @@ --- name: ruby-3.3 +dependencies: +- rust-1-linux + files: - compile-3.3.env - config/config.guess diff --git a/packages/ruby-3.4/packaging b/packages/ruby-3.4/packaging index 02c22a7..92d5141 100644 --- a/packages/ruby-3.4/packaging +++ b/packages/ruby-3.4/packaging @@ -12,17 +12,17 @@ if [ $(uname -s) == "Darwin" ]; then PATH="${brew_prefix_openssl}/bin:$PATH" with_openssl_dir="--with-openssl-dir=$brew_prefix_openssl" fi -fi +else + export CC="gcc" + export CXX="g++" -# Use Clang if available; the resulting Ruby is faster -if command -v clang &> /dev/null -then - export CC="$(command -v clang)" -fi + echo "GCC version:" + $CC --version + + source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/rust-1-linux/bosh/compile.env" -if command -v clang++ &> /dev/null -then - export CXX="$(command -v clang++)" + echo "Rust version:" + rustc --version --verbose fi openssl_version=$(openssl version) @@ -58,6 +58,12 @@ tar xzf "ruby-${RUBY_VERSION}.tar.gz" cp "${BOSH_COMPILE_TARGET}"/config/config.{guess,sub} ./tool fi + if [ "$(uname -s)" != "Darwin" ]; then + # GNU ld cannot handle LLVM bitcode objects produced by thin LTO; disable it + # so Rust emits plain native objects that GCC can link without issues. + sed -i 's/-C lto=thin/-C lto=no/g' common.mk + fi + LDFLAGS="-Wl,-rpath -Wl,${BOSH_INSTALL_TARGET}" ./configure --prefix="${BOSH_INSTALL_TARGET}" --disable-install-doc --with-opt-dir="${BOSH_INSTALL_TARGET}" --without-gmp $with_openssl_dir make make install diff --git a/packages/ruby-3.4/spec b/packages/ruby-3.4/spec index e6da5cf..63557a4 100644 --- a/packages/ruby-3.4/spec +++ b/packages/ruby-3.4/spec @@ -1,6 +1,9 @@ --- name: ruby-3.4 +dependencies: +- rust-1-linux + files: - compile-3.4.env - config/config.guess diff --git a/packages/ruby-4.0/packaging b/packages/ruby-4.0/packaging index c91476b..0162a20 100644 --- a/packages/ruby-4.0/packaging +++ b/packages/ruby-4.0/packaging @@ -12,17 +12,17 @@ if [ $(uname -s) == "Darwin" ]; then PATH="${brew_prefix_openssl}/bin:$PATH" with_openssl_dir="--with-openssl-dir=$brew_prefix_openssl" fi -fi +else + export CC="gcc" + export CXX="g++" -# Use Clang if available; the resulting Ruby is faster -if command -v clang &> /dev/null -then - export CC="$(command -v clang)" -fi + echo "GCC version:" + $CC --version + + source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/rust-1-linux/bosh/compile.env" -if command -v clang++ &> /dev/null -then - export CXX="$(command -v clang++)" + echo "Rust version:" + rustc --version --verbose fi openssl_version=$(openssl version) @@ -58,6 +58,12 @@ tar xzf "ruby-${RUBY_VERSION}.tar.gz" cp "${BOSH_COMPILE_TARGET}"/config/config.{guess,sub} ./tool fi + if [ "$(uname -s)" != "Darwin" ]; then + # GNU ld cannot handle LLVM bitcode objects produced by thin LTO; disable it + # so Rust emits plain native objects that GCC can link without issues. + sed -i 's/-C lto=thin/-C lto=no/g' common.mk + fi + LDFLAGS="-Wl,-rpath -Wl,${BOSH_INSTALL_TARGET}" ./configure --prefix="${BOSH_INSTALL_TARGET}" --disable-install-doc --with-opt-dir="${BOSH_INSTALL_TARGET}" --without-gmp $with_openssl_dir make make install diff --git a/packages/ruby-4.0/spec b/packages/ruby-4.0/spec index 0158bfb..bd83aa0 100644 --- a/packages/ruby-4.0/spec +++ b/packages/ruby-4.0/spec @@ -1,6 +1,9 @@ --- name: ruby-4.0 +dependencies: +- rust-1-linux + files: - compile-4.0.env - config/config.guess diff --git a/packages/rust-1-linux/spec.lock b/packages/rust-1-linux/spec.lock new file mode 100644 index 0000000..5c00338 --- /dev/null +++ b/packages/rust-1-linux/spec.lock @@ -0,0 +1,2 @@ +name: rust-1-linux +fingerprint: b670803b2abcaea7e51ef668e638bc2da755df840759a1c285c2ccefd5aecbfa diff --git a/src/compile-3.3.env b/src/compile-3.3.env index 87ec140..13a7565 100644 --- a/src/compile-3.3.env +++ b/src/compile-3.3.env @@ -3,17 +3,6 @@ # shellcheck disable=1090 source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/${PACKAGE_NAME}/bosh/runtime.env" -# Use Clang if available; the resulting Ruby is faster -if command -v clang &> /dev/null; then - CC="$(command -v clang)" - export CC -fi - -if command -v clang++ &> /dev/null; then - CXX="$(command -v clang++)" - export CXX -fi - bosh_bundle() { bundle config set --local no_prune 'true' bundle config set --local without 'development test' diff --git a/src/compile-3.4.env b/src/compile-3.4.env index d1c8b60..6d18a22 100644 --- a/src/compile-3.4.env +++ b/src/compile-3.4.env @@ -3,17 +3,6 @@ # shellcheck disable=1090 source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/${PACKAGE_NAME}/bosh/runtime.env" -# Use Clang if available; the resulting Ruby is faster -if command -v clang &> /dev/null; then - CC="$(command -v clang)" - export CC -fi - -if command -v clang++ &> /dev/null; then - CXX="$(command -v clang++)" - export CXX -fi - bosh_bundle() { bundle config set --local no_prune 'true' bundle config set --local without 'development test' diff --git a/src/compile-4.0.env b/src/compile-4.0.env index 8368b98..122d71b 100644 --- a/src/compile-4.0.env +++ b/src/compile-4.0.env @@ -3,17 +3,6 @@ # shellcheck disable=1090 source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/${PACKAGE_NAME}/bosh/runtime.env" -# Use Clang if available; the resulting Ruby is faster -if command -v clang &> /dev/null; then - CC="$(command -v clang)" - export CC -fi - -if command -v clang++ &> /dev/null; then - CXX="$(command -v clang++)" - export CXX -fi - bosh_bundle() { bundle config set --local no_prune 'true' bundle config set --local without 'development test'