Add rust-1-linux dependency to all ruby packages for YJIT/ZJIT

ci/pipeline.yml

@@ -127,6 +127,95 @@ jobs:
         repository: release_repo
         tag: release_metadata/tag-name
 #@ end
+- name: bump-rust-1-linux
+  serial: true
+  serial_groups:
+    - bump
+  plan:
+  - in_parallel:
+    - get: rust-release
+      trigger: true
+    - get: ruby-release
+    - get: stemcell
+      resource: ubuntu-noble
+    - get: bosh-src
+    - get: semver
+      params:
+        bump: minor
+    - get: bosh-shared-ci
+    - get: docker-cpi-image
+  - task: bump-rust-package
+    file: rust-release/ci/tasks/shared/bump-rust-package/task.yml
+    image: docker-cpi-image
+    input_mapping:
+      release-repo: ruby-release
+    params:
+      GIT_USER_NAME: CI Bot
+      GIT_USER_EMAIL: bots@cloudfoundry.org
+      PACKAGE: rust-1-linux
+      PRIVATE_YML: |
+        ---
+        blobstore:
+          provider: gcs
+          options:
+            credentials_source: static
+            json_key: '((gcp_json_key))'
+  - in_parallel:
+    - task: test-3.3
+      file: ruby-release/ci/tasks/test.yml
+      image: docker-cpi-image
+      privileged: true
+      timeout: 2h
+      input_mapping:
+        ruby-release: release-repo
+      params:
+        RUBY_VERSION: "3.3"
+        STEMCELL: ubuntu-noble
+    - task: test-3.4
+      file: ruby-release/ci/tasks/test.yml
+      image: docker-cpi-image
+      privileged: true
+      timeout: 2h
+      input_mapping:
+        ruby-release: release-repo
+      params:
+        RUBY_VERSION: "3.4"
+        STEMCELL: ubuntu-noble
+    - task: test-4.0
+      file: ruby-release/ci/tasks/test.yml
+      image: docker-cpi-image
+      privileged: true
+      timeout: 2h
+      input_mapping:
+        ruby-release: release-repo
+      params:
+        RUBY_VERSION: "4.0"
+        STEMCELL: ubuntu-noble
+  - task: finalize
+    file: bosh-shared-ci/tasks/release/create-final-release.yml
+    input_mapping:
+      release_repo: release-repo
+      version: semver
+    params:
+      GIT_USER_NAME: CI Bot
+      GIT_USER_EMAIL: bots@cloudfoundry.org
+      PRIVATE_YML: |
+        ---
+        blobstore:
+          provider: gcs
+          options:
+            credentials_source: static
+            json_key: '((gcp_json_key))'
+  - in_parallel:
+    - put: semver
+      params:
+        file: semver/version
+    - put: ruby-release
+      params:
+        rebase: true
+        repository: release_repo
+        tag: release_metadata/tag-name
+
 - name: build-docker-image
   plan:
   - in_parallel:
@@ -177,6 +266,14 @@ resources:
   source:
     uri: https://github.com/cloudfoundry/bosh.git
 
+- name: rust-release
+  type: git
+  source:
+    uri: https://github.com/cloudfoundry/bosh-package-rust-release.git
+    branch: main
+    paths:
+      - .final_builds/packages/rust-1-linux/index.yml
+
 - name: ruby-release
   type: git
   source:

ci/templates/jobs/ruby-test/templates/run.erb

@@ -38,3 +38,21 @@ else
   echo "wrong rubygems version: $gems_version"
   exit 1
 fi
+
+echo "Testing that Ruby was compiled with Rust (YJIT support)"
+
+if ruby -e 'exit(1) unless defined?(RubyVM::YJIT)'; then
+  echo "YJIT is compiled into Ruby"
+else
+  echo "ERROR: YJIT is NOT compiled into Ruby. Was Rust available during compilation?"
+  exit 1
+fi
+
+echo "Testing that YJIT can be enabled at runtime"
+
+if ruby --yjit -e 'RubyVM::YJIT.enable; raise "YJIT not enabled" unless RubyVM::YJIT.enabled?; puts "YJIT enabled successfully"'; then
+  echo "YJIT is functional"
+else
+  echo "ERROR: YJIT could not be enabled"
+  exit 1
+fi

ci/templates/packages/ruby/packaging.erb

@@ -12,17 +12,17 @@ if [ $(uname -s) == "Darwin" ]; then
     PATH="${brew_prefix_openssl}/bin:$PATH"
     with_openssl_dir="--with-openssl-dir=$brew_prefix_openssl"
   fi
-fi
+else
+  export CC="gcc"
+  export CXX="g++"
 
-# Use Clang if available; the resulting Ruby is faster
-if command -v clang &> /dev/null
-then
-  export CC="$(command -v clang)"
-fi
+  echo "GCC version:"
+  $CC --version
+
+  source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/rust-1-linux/bosh/compile.env"
 
-if command -v clang++ &> /dev/null
-then
-  export CXX="$(command -v clang++)"
+  echo "Rust version:"
+  rustc --version --verbose
 fi
 
 openssl_version=$(openssl version)
@@ -58,6 +58,12 @@ tar xzf "ruby-${RUBY_VERSION}.tar.gz"
     cp "${BOSH_COMPILE_TARGET}"/config/config.{guess,sub} ./tool
   fi
 
+  if [ "$(uname -s)" != "Darwin" ]; then
+    # GNU ld cannot handle LLVM bitcode objects produced by thin LTO; disable it
+    # so Rust emits plain native objects that GCC can link without issues.
+    sed -i 's/-C lto=thin/-C lto=no/g' common.mk
+  fi
+
   LDFLAGS="-Wl,-rpath -Wl,${BOSH_INSTALL_TARGET}" ./configure --prefix="${BOSH_INSTALL_TARGET}" --disable-install-doc --with-opt-dir="${BOSH_INSTALL_TARGET}" --without-gmp $with_openssl_dir
   make
   make install

ci/templates/packages/ruby/spec.erb

@@ -1,6 +1,9 @@
 ---
 name: <%= ruby_package_name %>
 
+dependencies:
+- rust-1-linux
+
 files:
 - compile-<%= ruby_version %>.env
 - config/config.guess

ci/templates/src/compile.env.erb

@@ -3,17 +3,6 @@
 # shellcheck disable=1090
 source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/${PACKAGE_NAME}/bosh/runtime.env"
 
-# Use Clang if available; the resulting Ruby is faster
-if command -v clang &> /dev/null; then
-  CC="$(command -v clang)"
-  export CC
-fi
-
-if command -v clang++ &> /dev/null; then
-  CXX="$(command -v clang++)"
-  export CXX
-fi
-
 bosh_bundle() {
   bundle config set --local no_prune 'true'
   bundle config set --local without 'development test'

config/blobs.yml

@@ -22,6 +22,9 @@ rubygems-4.0.16.tgz:
   size: 978912
   object_id: 2a34dec1-5ae5-45eb-5f4d-bf98503ce743
   sha: sha256:ea9c669526af82874f8f33f69bea1b6ddd99283756e598227a9a890035a5a06a
+rust-1-linux/rust-1.97.0-x86_64-unknown-linux-gnu.tar.gz:
+  size: 384617996
+  sha: sha256:eb89b20287153391c49ebcdb7fd91b683a12438d129bfb92eadcc495545af3a7
 yaml-0.2.5.tar.gz:
   size: 609454
   object_id: b5eeae2b-1496-4bc2-556f-efac50c8bcfa

jobs/ruby-3.3-test/templates/run

@@ -38,3 +38,21 @@ else
   echo "wrong rubygems version: $gems_version"
   exit 1
 fi
+
+echo "Testing that Ruby was compiled with Rust (YJIT support)"
+
+if ruby -e 'exit(1) unless defined?(RubyVM::YJIT)'; then
+  echo "YJIT is compiled into Ruby"
+else
+  echo "ERROR: YJIT is NOT compiled into Ruby. Was Rust available during compilation?"
+  exit 1
+fi
+
+echo "Testing that YJIT can be enabled at runtime"
+
+if ruby --yjit -e 'RubyVM::YJIT.enable; raise "YJIT not enabled" unless RubyVM::YJIT.enabled?; puts "YJIT enabled successfully"'; then
+  echo "YJIT is functional"
+else
+  echo "ERROR: YJIT could not be enabled"
+  exit 1
+fi

jobs/ruby-3.4-test/templates/run

@@ -38,3 +38,21 @@ else
   echo "wrong rubygems version: $gems_version"
   exit 1
 fi
+
+echo "Testing that Ruby was compiled with Rust (YJIT support)"
+
+if ruby -e 'exit(1) unless defined?(RubyVM::YJIT)'; then
+  echo "YJIT is compiled into Ruby"
+else
+  echo "ERROR: YJIT is NOT compiled into Ruby. Was Rust available during compilation?"
+  exit 1
+fi
+
+echo "Testing that YJIT can be enabled at runtime"
+
+if ruby --yjit -e 'RubyVM::YJIT.enable; raise "YJIT not enabled" unless RubyVM::YJIT.enabled?; puts "YJIT enabled successfully"'; then
+  echo "YJIT is functional"
+else
+  echo "ERROR: YJIT could not be enabled"
+  exit 1
+fi

jobs/ruby-4.0-test/templates/run

@@ -38,3 +38,21 @@ else
   echo "wrong rubygems version: $gems_version"
   exit 1
 fi
+
+echo "Testing that Ruby was compiled with Rust (YJIT support)"
+
+if ruby -e 'exit(1) unless defined?(RubyVM::YJIT)'; then
+  echo "YJIT is compiled into Ruby"
+else
+  echo "ERROR: YJIT is NOT compiled into Ruby. Was Rust available during compilation?"
+  exit 1
+fi
+
+echo "Testing that YJIT can be enabled at runtime"
+
+if ruby --yjit -e 'RubyVM::YJIT.enable; raise "YJIT not enabled" unless RubyVM::YJIT.enabled?; puts "YJIT enabled successfully"'; then
+  echo "YJIT is functional"
+else
+  echo "ERROR: YJIT could not be enabled"
+  exit 1
+fi

packages/ruby-3.3/packaging

@@ -12,17 +12,17 @@ if [ $(uname -s) == "Darwin" ]; then
     PATH="${brew_prefix_openssl}/bin:$PATH"
     with_openssl_dir="--with-openssl-dir=$brew_prefix_openssl"
   fi
-fi
+else
+  export CC="gcc"
+  export CXX="g++"
 
-# Use Clang if available; the resulting Ruby is faster
-if command -v clang &> /dev/null
-then
-  export CC="$(command -v clang)"
-fi
+  echo "GCC version:"
+  $CC --version
+
+  source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/rust-1-linux/bosh/compile.env"
 
-if command -v clang++ &> /dev/null
-then
-  export CXX="$(command -v clang++)"
+  echo "Rust version:"
+  rustc --version --verbose
 fi
 
 openssl_version=$(openssl version)
@@ -58,6 +58,12 @@ tar xzf "ruby-${RUBY_VERSION}.tar.gz"
     cp "${BOSH_COMPILE_TARGET}"/config/config.{guess,sub} ./tool
   fi
 
+  if [ "$(uname -s)" != "Darwin" ]; then
+    # GNU ld cannot handle LLVM bitcode objects produced by thin LTO; disable it
+    # so Rust emits plain native objects that GCC can link without issues.
+    sed -i 's/-C lto=thin/-C lto=no/g' common.mk
+  fi
+
   LDFLAGS="-Wl,-rpath -Wl,${BOSH_INSTALL_TARGET}" ./configure --prefix="${BOSH_INSTALL_TARGET}" --disable-install-doc --with-opt-dir="${BOSH_INSTALL_TARGET}" --without-gmp $with_openssl_dir
   make
   make install

packages/ruby-3.3/spec

@@ -1,6 +1,9 @@
 ---
 name: ruby-3.3
 
+dependencies:
+- rust-1-linux
+
 files:
 - compile-3.3.env
 - config/config.guess

packages/ruby-3.4/packaging

@@ -12,17 +12,17 @@ if [ $(uname -s) == "Darwin" ]; then
     PATH="${brew_prefix_openssl}/bin:$PATH"
     with_openssl_dir="--with-openssl-dir=$brew_prefix_openssl"
   fi
-fi
+else
+  export CC="gcc"
+  export CXX="g++"
 
-# Use Clang if available; the resulting Ruby is faster
-if command -v clang &> /dev/null
-then
-  export CC="$(command -v clang)"
-fi
+  echo "GCC version:"
+  $CC --version
+
+  source "${BOSH_PACKAGES_DIR:-/var/vcap/packages}/rust-1-linux/bosh/compile.env"
 
-if command -v clang++ &> /dev/null
-then
-  export CXX="$(command -v clang++)"
+  echo "Rust version:"
+  rustc --version --verbose
 fi
 
 openssl_version=$(openssl version)
@@ -58,6 +58,12 @@ tar xzf "ruby-${RUBY_VERSION}.tar.gz"
     cp "${BOSH_COMPILE_TARGET}"/config/config.{guess,sub} ./tool
   fi
 
+  if [ "$(uname -s)" != "Darwin" ]; then
+    # GNU ld cannot handle LLVM bitcode objects produced by thin LTO; disable it
+    # so Rust emits plain native objects that GCC can link without issues.
+    sed -i 's/-C lto=thin/-C lto=no/g' common.mk
+  fi
+
   LDFLAGS="-Wl,-rpath -Wl,${BOSH_INSTALL_TARGET}" ./configure --prefix="${BOSH_INSTALL_TARGET}" --disable-install-doc --with-opt-dir="${BOSH_INSTALL_TARGET}" --without-gmp $with_openssl_dir
   make
   make install

packages/ruby-3.4/spec

@@ -1,6 +1,9 @@
 ---
 name: ruby-3.4
 
+dependencies:
+- rust-1-linux
+
 files:
 - compile-3.4.env
 - config/config.guess