diff --git a/o/opensearch-project-ml-commons/build_info.json b/o/opensearch-project-ml-commons/build_info.json new file mode 100644 index 0000000000..62dd474669 --- /dev/null +++ b/o/opensearch-project-ml-commons/build_info.json @@ -0,0 +1,25 @@ +{ + "maintainer": "Prachi.Gaonkar@ibm.com", + "package_name": "opensearch-project-ml-commons", + "github_url": "https://github.com/opensearch-project/ml-commons", + "version": "3.5.0.0", + "required_versions":{"Releases": ["*"], "Tags": ["*"]}, + "default_branch": "main", + "package_dir": "o/opensearch-project-ml-commons", + "build_script": "ml-commons_3.5.0.0_ubi9.6.sh", + "validate_build_script": true, + "use_non_root_user": true, + "docker_build": false, + "3.5.0.0": { + "build_script": "ml-commons_3.5.0.0_ubi9.6.sh" + }, + "3.3.0.0": { + "build_script": "ml-commons_3.3.0.0_ubi9.6.sh" + }, + "2.19.2.0": { + "build_script": "ml-commons_2.19.2.0_ubi9.5.sh" + }, + "*.*.*.*": { + "build_script": "ml-commons_3.5.0.0_ubi9.6.sh" + } +} \ No newline at end of file diff --git a/o/opensearch-project-ml-commons/djl_v0.33.0_updated.patch b/o/opensearch-project-ml-commons/djl_v0.33.0_updated.patch new file mode 100644 index 0000000000..2ee45fc08f --- /dev/null +++ b/o/opensearch-project-ml-commons/djl_v0.33.0_updated.patch @@ -0,0 +1,133 @@ +diff --git a/engines/pytorch/pytorch-jni/build.gradle.kts b/engines/pytorch/pytorch-jni/build.gradle.kts +index c2c0c0c84..72c08b62a 100644 +--- a/engines/pytorch/pytorch-jni/build.gradle.kts ++++ b/engines/pytorch/pytorch-jni/build.gradle.kts +@@ -29,33 +29,47 @@ tasks { + val hasJni = project.hasProperty("jni") + + doFirst { +- val url = "https://publish.djl.ai/pytorch/$ptVersion/jnilib/${libs.versions.djl.get()}" +- val files = listOf( +- "linux-x86_64/cpu/libdjl_torch.so", +- "linux-x86_64/cpu-precxx11/libdjl_torch.so", +- "linux-aarch64/cpu-precxx11/libdjl_torch.so", +- "osx-aarch64/cpu/libdjl_torch.dylib", +- "win-x86_64/cpu/djl_torch.dll" +- ) + when { +- ptVersion.matches(Regex("2.[4-5].\\d")) -> listOf( +- "linux-x86_64/cu124/libdjl_torch.so", +- "linux-x86_64/cu124-precxx11/libdjl_torch.so", +- "win-x86_64/cu124/djl_torch.dll" +- ) +- +- ptVersion.matches(Regex("2.[1-3].\\d")) -> listOf( +- "linux-x86_64/cu121/libdjl_torch.so", +- "linux-x86_64/cu121-precxx11/libdjl_torch.so", +- "win-x86_64/cu121/djl_torch.dll", +- ) +- +- ptVersion.startsWith("1.13.") -> listOf( +- "linux-x86_64/cu117/libdjl_torch.so", +- "win-x86_64/cu117/djl_torch.dll", +- ) +- +- else -> throw GradleException("Unsupported version: $ptVersion.") +- } ++ val url = "https://publish.djl.ai/pytorch/$ptVersion/jnilib/${libs.versions.djl.get()}" ++ val arch = System.getProperty("os.arch") ++ ++ val files = when (arch) { ++ "ppc64le" -> listOf( ++ "linux-ppc64le/cpu/libdjl_torch.so" ++ ) ++ ++ "x86_64", "amd64" -> listOf( ++ "linux-x86_64/cpu/libdjl_torch.so", ++ "linux-x86_64/cpu-precxx11/libdjl_torch.so" ++ ) + when { ++ ptVersion.matches(Regex("2.[4-9].\\d")) -> listOf( ++ "linux-x86_64/cu124/libdjl_torch.so", ++ "linux-x86_64/cu124-precxx11/libdjl_torch.so", ++ "win-x86_64/cu124/djl_torch.dll" ++ ) ++ ++ ptVersion.matches(Regex("2.[1-3].\\d")) -> listOf( ++ "linux-x86_64/cu121/libdjl_torch.so", ++ "linux-x86_64/cu121-precxx11/libdjl_torch.so", ++ "win-x86_64/cu121/djl_torch.dll" ++ ) ++ ++ ptVersion.startsWith("1.13.") -> listOf( ++ "linux-x86_64/cu117/libdjl_torch.so", ++ "win-x86_64/cu117/djl_torch.dll" ++ ) ++ ++ else -> emptyList() ++ } ++ ++ "aarch64" -> listOf( ++ "linux-aarch64/cpu-precxx11/libdjl_torch.so", ++ "osx-aarch64/cpu/libdjl_torch.dylib" ++ ) ++ ++ else -> throw GradleException("Unsupported arch: $arch") ++ ++ } ++ + val jnilibDir = dir / "jnilib" / libs.versions.djl.get() + for (entry in files) { + val file = jnilibDir / entry +diff --git a/engines/pytorch/pytorch-native/build.gradle.kts b/engines/pytorch/pytorch-native/build.gradle.kts +index e5f002961..7d877e93a 100644 +--- a/engines/pytorch/pytorch-native/build.gradle.kts ++++ b/engines/pytorch/pytorch-native/build.gradle.kts +@@ -33,7 +33,7 @@ fun downloadBuild( + isPrecxx11: Boolean = false, + isAarch64: Boolean = false + ) { +- val arch = if (isAarch64) "aarch64" else "x86_64" ++ val arch = if (isAarch64) "aarch64" else "ppc64le" + execOperations.exec { + workingDir = project.projectDir + if (os == "win") +@@ -92,6 +92,7 @@ fun prepareNativeLib(execOperations: ExecOperations, binaryRoot: String, ver: St + } else { + // @formatter:off + val files = mapOf( ++ "cpu/libtorch-cxx11-abi-shared-with-deps-$ver%2Bcpu.zip" to "cpu/linux-ppc64le", + "cpu/libtorch-cxx11-abi-shared-with-deps-$ver%2Bcpu.zip" to "cpu/linux-x86_64", + "cpu/libtorch-macos-arm64-$ver.zip" to "cpu/osx-aarch64", + "cpu/libtorch-win-shared-with-deps-$ver%2Bcpu.zip" to "cpu/win-x86_64", +@@ -256,6 +257,7 @@ tasks { + + (binaryRoot / "files.txt").text = buildString { + val uploadDirs = listOf( ++ binaryRoot / "cpu/linux-ppc64le/native/lib/", + binaryRoot / "cpu/linux-x86_64/native/lib/", + binaryRoot / "cpu/osx-aarch64/native/lib/", + binaryRoot / "cpu/win-x86_64/native/lib/", +diff --git a/extensions/tokenizers/build.gradle.kts b/extensions/tokenizers/build.gradle.kts +index 8108d174d..05adfb973 100644 +--- a/extensions/tokenizers/build.gradle.kts ++++ b/extensions/tokenizers/build.gradle.kts +@@ -52,6 +52,7 @@ tasks { + "win-x86_64/cpu/libgcc_s_seh-1.dll" to "extra/win-x86_64/libgcc_s_seh-1.dll", + "win-x86_64/cpu/libstdc%2B%2B-6.dll" to "extra/win-x86_64/libstdc%2B%2B-6.dll", + "win-x86_64/cpu/tokenizers.dll" to "$tokenizers/jnilib/$djl", ++ "linux-ppc64le/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl", + "linux-x86_64/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl", + "linux-aarch64/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl", + "osx-aarch64/cpu/libtokenizers.dylib" to "$tokenizers/jnilib/$djl" +diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml +index fcb16e641..d6db2877b 100644 +--- a/gradle/libs.versions.toml ++++ b/gradle/libs.versions.toml +@@ -3,7 +3,7 @@ format.version = "1.1" + + [versions] + djl = "0.33.0" +-pytorch = "2.5.1" ++pytorch = "2.9.1" + tensorflow = "2.16.1" + tensorflowCore = "1.0.0-rc.1" + mxnet = "1.9.1" diff --git a/o/opensearch-project-ml-commons/ml-commons_3.3.0.0_ubi9.6.sh b/o/opensearch-project-ml-commons/ml-commons_3.3.0.0_ubi9.6.sh index 0e307ab7a3..75179bbc98 100644 --- a/o/opensearch-project-ml-commons/ml-commons_3.3.0.0_ubi9.6.sh +++ b/o/opensearch-project-ml-commons/ml-commons_3.3.0.0_ubi9.6.sh @@ -166,7 +166,7 @@ wget https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/master/o git clone https://github.com/deepjavalibrary/djl cd djl/ git checkout $DJL_VERSION -git apply $BUILD_HOME/djl_$DJL_VERSION.patch +git apply $BUILD_HOME/djl_$DJL_VERSION_updated.patch wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip unzip libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}+cpu.zip -d $BUILD_HOME/djl/engines/pytorch/pytorch-native rm -rf libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}+cpu.zip @@ -249,7 +249,7 @@ git apply ${SCRIPT_PATH}/ml-commons_$SCRIPT_PACKAGE_VERSION.patch # Build # -------- ret=0 -./gradlew build -x test -x integTest -Dbuild.snapshot=false|| ret=$? +./gradlew build -x test -x integTest || ret=$? if [ $ret -ne 0 ]; then set +ex echo "------------------ ${PACKAGE_NAME}: Build Failed ------------------" diff --git a/o/opensearch-project-ml-commons/ml-commons_3.5.0.0_ubi9.6.sh b/o/opensearch-project-ml-commons/ml-commons_3.5.0.0_ubi9.6.sh new file mode 100644 index 0000000000..11336e0991 --- /dev/null +++ b/o/opensearch-project-ml-commons/ml-commons_3.5.0.0_ubi9.6.sh @@ -0,0 +1,335 @@ +#!/bin/bash -ex +# -------------------------------------------------------------------------------- +# Package : ml-commons +# Version : 3.5.0.0 +# Source repo : https://github.com/opensearch-project/ml-commons +# Tested on : UBI 9.6 +# Language : Java +# Ci-Check : True +# Maintainer : Prachi Gaonkar +# Script License : Apache License, Version 2.0 or later +# +# Disclaimer : This script has been tested in non root mode on the specified +# platform and package version. Functionality with newer +# versions of the package or OS is not guaranteed. +# ------------------------------------------------------------------------------- + +# --------------------------- +# Check for root user +# --------------------------- +if [[ "$EUID" -eq 0 ]]; then + set +ex + echo "FAIL: Run this script as a non-root user with sudo permissions" + exit 3 +fi + + +# --------------------------- +# Configuration +# --------------------------- +PACKAGE_NAME="ml-commons" +PACKAGE_ORG="opensearch-project" +SCRIPT_PACKAGE_VERSION="3.5.0.0" +PACKAGE_VERSION="${1:-$SCRIPT_PACKAGE_VERSION}" +PACKAGE_URL="https://github.com/${PACKAGE_ORG}/${PACKAGE_NAME}.git" +OPENSEARCH_VERSION="${PACKAGE_VERSION::-2}" +OPENSEARCH_PACKAGE="OpenSearch" +OPENSEARCH_URL=https://github.com/${PACKAGE_ORG}/${OPENSEARCH_PACKAGE}.git +ONNX_VERSION="v1.17.1" +PYTORCH_VERSION=2.9.1 +PYTHON_VERSION=3.12 +CMAKE_VERSION=3.28.1 +DJL_VERSION="v0.33.0" +BUILD_HOME="$(pwd)" +SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DJL_HOME="$HOME/.djl.ai" +RUN_TESTS=1 + +sudo chown -R test_user:test_user /home/tester +# ------------------- +# Parse CLI Arguments +# ------------------- +for i in "$@"; do + case $i in + --skip-tests) + RUN_TESTS=0 + echo "Skipping tests" + shift + ;; + -*|--*) + echo "Unknown option $i" + exit 3 + ;; + *) + PACKAGE_VERSION=$i + echo "Building ${PACKAGE_NAME} ${PACKAGE_VERSION}" + ;; + esac +done + +# --------------------------- +# Dependency Installation +# --------------------------- +# Installs both JDK 17 and JDK 21 as required by different components +sudo yum config-manager --add-repo https://mirror.stream.centos.org/9-stream/CRB/ppc64le/os +sudo yum config-manager --add-repo https://mirror.stream.centos.org/9-stream/AppStream//ppc64le/os +sudo yum config-manager --add-repo https://mirror.stream.centos.org/9-stream/BaseOS/ppc64le/os +sudo rpm --import https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official-SHA256 +sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm +sudo rpm -e --nodeps openssl-fips-provider-so +sudo yum install -y git wget unzip make procps-ng perl python3.12-devel python3.12-pip java-21-openjdk-devel java-17-openjdk-devel openblas-devel bzip2-devel zlib-devel protobuf-devel ncurses-devel clang15 llvm15 lld +export CC=clang-15 +export CXX=clang++-15 + + +# -------------------------------------- +#Use jdk21 for ml-commons and djl +# -------------------------------------- +export JAVA_HOME=$(compgen -G '/usr/lib/jvm/java-21-openjdk-*') +export JRE_HOME=${JAVA_HOME}/jre +export PATH=${JAVA_HOME}/bin:$PATH + +CMAKE_PREFIX=/usr/local/cmake +CMAKE_BIN=${CMAKE_PREFIX}/bin/cmake + +# --------------------------- +# Build and Install CMake +# --------------------------- +# --------------------------- +# Build and Install cmake +# --------------------------- +cd $BUILD_HOME +if [ -z "$(ls -A $BUILD_HOME/cmake-${CMAKE_VERSION})" ]; then + wget -c https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz + tar -zxvf cmake-${CMAKE_VERSION}.tar.gz + rm -rf cmake-${CMAKE_VERSION}.tar.gz + cd cmake-${CMAKE_VERSION} + ./bootstrap --prefix=/usr/local/cmake --parallel=2 -- -DBUILD_TESTING:BOOL=OFF -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_USE_OPENSSL:BOOL=OFF +else + cd cmake-${CMAKE_VERSION} +fi +sudo make install -j$(nproc) +export PATH=/usr/local/cmake/bin:$PATH +cmake --version + + + + +export JAVA_HOME=$(compgen -G '/usr/lib/jvm/java-21-openjdk-*') +export JRE_HOME=${JAVA_HOME}/jre +export PATH=${JAVA_HOME}/bin:$PATH +# -------------- +# Install rust +# -------------- +curl https://sh.rustup.rs -sSf | sh -s -- -y +source $HOME/.cargo/env +rustup install 1.87 +rustup default 1.87 + + +# --------------------------- +# Build and Install pytorch +# --------------------------- +cd $BUILD_HOME +PYTORCH_VERSION_CPU="${PYTORCH_VERSION}+cpu" +export PYTORCH_BUILD_VERSION=${PYTORCH_VERSION_CPU} +if [ -z "$(ls -A $BUILD_HOME/pytorch)" ]; then + git clone https://github.com/pytorch/pytorch + cd pytorch + git checkout v$PYTORCH_VERSION + + git submodule sync + git submodule update --init --recursive + + git apply --check ${SCRIPT_PATH}/pytorch_vec_splat_issue_fix.patch + + git apply ${SCRIPT_PATH}/pytorch_vec_splat_issue_fix.patch + + + python3.12 -m pip install -r requirements.txt + export PYTORCH_BUILD_NUMBER=1 + export CFLAGS="-mcpu=power9 -mtune=power9" + export CXXFLAGS="-mcpu=power9 -mtune=power9" + python3.12 setup.py bdist_wheel +else + cd pytorch +fi +python3.12 -m pip install $BUILD_HOME/pytorch/dist/torch-${PYTORCH_BUILD_VERSION}*.whl + +# --------------------------- +# Python native dependencies for DJL +# --------------------------- +python3.12 -m pip install abseil_cpp==20240116.2 \ + --prefer-binary \ + --extra-index-url https://wheels.developerfirst.ibm.com/ppc64le/linux + +python3.12 -m pip install libprotobuf==4.25.3 \ + --prefer-binary \ + --extra-index-url=https://wheels.developerfirst.ibm.com/ppc64le/linux + +# ---------------------- +# Build and Install djl +# ---------------------- +cd $BUILD_HOME +git clone https://github.com/deepjavalibrary/djl +cd djl/ +git checkout $DJL_VERSION +git apply ${SCRIPT_PATH}/djl_${DJL_VERSION}_updated.patch +# Copy Python3.12 torch binaries +rm -rf $BUILD_HOME/djl/engines/pytorch/pytorch-native/libtorch/include +rm -rf $BUILD_HOME/djl/engines/pytorch/pytorch-native/libtorch/share +mkdir -p $BUILD_HOME/djl/engines/pytorch/pytorch-native/libtorch/lib/ +\cp -rf $HOME/.local/lib/python3.12/site-packages/torch/share $BUILD_HOME/djl/engines/pytorch/pytorch-native/libtorch/ +\cp -rf $HOME/.local/lib/python$PYTHON_VERSION/site-packages/torch/include $BUILD_HOME/djl/engines/pytorch/pytorch-native/libtorch/ +\cp -rf $HOME/.local/lib/python$PYTHON_VERSION/site-packages/torch/lib/*.so $BUILD_HOME/djl/engines/pytorch/pytorch-native/libtorch/lib/ +mkdir -p $DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le/ +cp $HOME/.local/lib/python$PYTHON_VERSION/site-packages/torch/lib/*.so $DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le/ +cp $HOME/.local/lib/python3.12/site-packages/libprotobuf/lib64/libprotobuf.so.25.3.0 $DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le +cp /usr/lib64/libopenblas.so.0 $DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le +cp /usr/lib64/libgfortran.so.5 $DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le +cp /usr/lib64/libquadmath.so.0 $DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le +\cp -rf $HOME/.local/lib/python3.12/site-packages/abseilcpp/lib/*.so $BUILD_HOME/djl/engines/pytorch/pytorch-native/libtorch/lib/ +\cp -rf $HOME/.local/lib/python3.12/site-packages/abseilcpp/lib/*.so $HOME/.djl.ai/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le + +# Create versioned symlinks for abseil libraries +cd $DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le/ + for f in libabsl_*.so; do ln -sf $f ${f}.2401.0.0; done + + +# --------------------------- +# Build DJL components +# --------------------------- +export LD_PRELOAD=$JAVA_HOME/lib/libjsig.so +cd $BUILD_HOME/djl +export LD_LIBRARY_PATH=$DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le:$LD_LIBRARY_PATH +./gradlew :engines:pytorch:pytorch-native:compileJNI + ./gradlew --no-daemon :engines:pytorch:pytorch-engine:test -Dengine.pytorch.disable_native_extraction=true -Djava.library.path=$DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le +./gradlew :extensions:tokenizers:compileJNI +./gradlew --no-daemon :extensions:tokenizers:test -Dengine.pytorch.disable_native_extraction=true -Djava.library.path=$DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le -Dai.djl.debug=true +./gradlew -Prelease=true publishToMavenLocal +cd bom +./gradlew build +./gradlew -Prelease=true publishToMavenLocal + + +# ------------------------------ +# Build OpenSearch distribution +# ------------------------------ +cd $BUILD_HOME +git clone https://github.com/opensearch-project/OpenSearch +cd OpenSearch +git checkout $OPENSEARCH_VERSION +./gradlew -p distribution/archives/linux-ppc64le-tar assemble +./gradlew -Prelease=true publishToMavenLocal +./gradlew :build-tools:publishToMavenLocal + +# --------------------------- +# Build Job Scheduler +# --------------------------- +cd $BUILD_HOME +git clone https://github.com/opensearch-project/job-scheduler +cd job-scheduler +git checkout $PACKAGE_VERSION +./gradlew assemble +./gradlew -Prelease=true publishToMavenLocal + +# --------------------------- +# Build Remote Metadata SDK +# --------------------------- +cd $BUILD_HOME +git clone https://github.com/opensearch-project/opensearch-remote-metadata-sdk +cd opensearch-remote-metadata-sdk +git checkout $PACKAGE_VERSION +export GRADLE_OPTS="-Dorg.gradle.console=plain" +./gradlew build +./gradlew -Prelease=true publishToMavenLocal + +# --------------------------------------------------------------------- +# Use jdk17 for ONNX Runtime +# --------------------------------------------------------------------- +export JAVA_HOME=$(compgen -G '/usr/lib/jvm/java-17-openjdk-*') +export JRE_HOME=${JAVA_HOME}/jre +export PATH=${JAVA_HOME}/bin:$PATH + +# -------------------------------------- +# Build ONNX Runtime with Java bindings +# -------------------------------------- +cd $BUILD_HOME +wget https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/master/o/opensearch-project-ml-commons/onnxruntime_$ONNX_VERSION.patch +git clone https://github.com/microsoft/onnxruntime.git +cd onnxruntime +git checkout $ONNX_VERSION +git apply $BUILD_HOME/onnxruntime_$ONNX_VERSION.patch +./build.sh --build_java --compile_no_warning_as_error --parallel --config=Release --build_shared_lib --skip_tests --allow_running_as_root +sudo cp $BUILD_HOME/onnxruntime/build/Linux/Release/libonnxruntime.so $BUILD_HOME/onnxruntime/build/Linux/Release/libonnxruntime4j_jni.so /usr/lib64/ + +# -------------------------------------- +#Use jdk21 for ml-commons and djl +# -------------------------------------- +export JAVA_HOME=$(compgen -G '/usr/lib/jvm/java-21-openjdk-*') +export JRE_HOME=${JAVA_HOME}/jre +export PATH=${JAVA_HOME}/bin:$PATH +# --------------------------- +# Clone and Prepare Repository +# --------------------------- +cd $BUILD_HOME +git clone https://github.com/opensearch-project/ml-commons +cd ml-commons +git checkout $PACKAGE_VERSION +git apply ${SCRIPT_PATH}/ml-commons_$SCRIPT_PACKAGE_VERSION.patch + + +# -------- +# Build +# -------- +ret=0 +./gradlew build -x test -x integTest -Dbuild.snapshot=false|| ret=$? +if [ $ret -ne 0 ]; then + set +ex + echo "------------------ ${PACKAGE_NAME}: Build Failed ------------------" + exit 1 +fi + +# -------- +# Install +# -------- +./gradlew -Prelease=true publishToMavenLocal + +# --------------------------- +# Skip Tests? +# --------------------------- +if [ "$RUN_TESTS" -eq 0 ]; then + set +ex + echo "------------------ Complete: Build and install successful! Tests skipped. ------------------" + exit 0 +fi + +# ---------- +# Unit Test +# ---------- +ret=0 +./gradlew test -x integTest --continue -Dorg.opensearch.djl.pytorch.path=$DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le || ret=$? +if [ $ret -ne 0 ]; then + ret=0 + ./gradlew test -x integTest || ret=$? + if [ $ret -ne 0 ]; then + set +ex + echo "------------------ ${PACKAGE_NAME}: Unit Test Failed ------------------" + exit 2 + fi +fi + +# ----------------- +# Integration Test +# ----------------- +ret=0 +./gradlew integTest -PcustomDistributionUrl=$BUILD_HOME/OpenSearch/distribution/archives/linux-ppc64le-tar/build/distributions/opensearch-min-$OPENSEARCH_VERSION-SNAPSHOT-linux-ppc64le.tar.gz -Dbuild.snapshot=false -Dorg.opensearch.djl.pytorch.path=$DJL_HOME/pytorch/$PYTORCH_VERSION-cpu-linux-ppc64le || ret=$? +if [ $ret -ne 0 ]; then + set +ex + echo "------------------ ${PACKAGE_NAME}: Integration Test Failed ------------------" + exit 2 +fi + +set +ex +echo "------------------ Complete: Build and Tests successful! ------------------" +echo "CI checks are disabled for this script due to the build time exceeding the maximum execution limit (6 hours) on GitHub Actions." \ No newline at end of file diff --git a/o/opensearch-project-ml-commons/pytorch_vec_splat_issue_fix.patch b/o/opensearch-project-ml-commons/pytorch_vec_splat_issue_fix.patch new file mode 100644 index 0000000000..94cb34399e --- /dev/null +++ b/o/opensearch-project-ml-commons/pytorch_vec_splat_issue_fix.patch @@ -0,0 +1,171 @@ +diff --git a/aten/src/ATen/cpu/vec/vec256/vsx/vec256_int64_vsx.h b/aten/src/ATen/cpu/vec/vec256/vsx/vec256_int64_vsx.h +index 8d0bd52c901..e4a062cdb4e 100644 +--- a/aten/src/ATen/cpu/vec/vec256/vsx/vec256_int64_vsx.h ++++ b/aten/src/ATen/cpu/vec/vec256/vsx/vec256_int64_vsx.h +@@ -3,6 +3,7 @@ + #include + #include + #include ++#include + namespace at { + namespace vec { + // See Note [CPU_CAPABILITY namespace] +@@ -42,7 +43,9 @@ class Vectorized { + C10_ALWAYS_INLINE Vectorized(vbool64 v1, vbool64 v2) + : _vecb0{v1}, _vecb1{v2} {} + C10_ALWAYS_INLINE Vectorized(int64_t scalar) +- : _vec0{vec_splats(scalar)}, _vec1{vec_splats(scalar)} {} ++ : _vec0{vec_splats_dispatch::apply(scalar)}, ++ _vec1{vec_splats_dispatch::apply(scalar)} {} ++ //: _vec0{vec_splats(scalar)}, _vec1{vec_splats(scalar)} {} + C10_ALWAYS_INLINE Vectorized( + int64_t scalar1, + int64_t scalar2, +@@ -296,6 +299,29 @@ operator^(const Vectorized& a, const Vectorized& b) { + vec_xor(a.vec0(), b.vec0()), vec_xor(a.vec1(), b.vec1())}; + } + ++#if defined(__powerpc64__) ++/* * We only define the 'long' specialization if it is a distinct type. ++ * On many PPC64LE Linux systems, int64_t is 'long'. ++ * On others (like some RHEL/CentOS versions), it is 'long long'. ++ * We use a macro-based check that is safe for the preprocessor. ++ */ ++#include ++#if defined(__GLIBC__) && defined(__WORDSIZE) && (__WORDSIZE == 64) ++ // In 64-bit GLIBC, int64_t is 'long'. We don't need a specialization. ++ #define INT64_IS_LONG 1 ++#else ++ #define INT64_IS_LONG 0 ++#endif ++ ++#if !INT64_IS_LONG ++template <> ++class Vectorized : public Vectorized { ++public: ++ using Vectorized::Vectorized; ++}; ++#endif ++#endif ++ + } // namespace CPU_CAPABILITY + } // namespace vec + } // namespace at +diff --git a/aten/src/ATen/cpu/vec/vec256/vsx/vsx_helpers.h b/aten/src/ATen/cpu/vec/vec256/vsx/vsx_helpers.h +index 7ca603c0b91..acf8315c165 100644 +--- a/aten/src/ATen/cpu/vec/vec256/vsx/vsx_helpers.h ++++ b/aten/src/ATen/cpu/vec/vec256/vsx/vsx_helpers.h +@@ -118,11 +118,62 @@ vec_sldw_aux(const vfloat32& vec_in0, const vfloat32& vec_in1) { + #endif + + #define vec_not(a) vec_nor(a, a) ++ ++// 1. General template (Works for GCC and for float/int32/etc.) ++template ++struct vec_splats_dispatch { ++ using vvtype = typename at::vec::Vectorized::vec_internal_type; ++ static C10_ALWAYS_INLINE vvtype apply(scalar_t v) { ++ return vec_splats(v); ++ } ++}; ++ ++// 2. Clang-specific override for 64-bit types ++#if defined(__clang__) && defined(__powerpc64__) ++template <> ++struct vec_splats_dispatch { ++ using vvtype = vint64; // Direct type to avoid circular dependency ++ static C10_ALWAYS_INLINE vvtype apply(int64_t v) { ++ // Your Power10 vec_insert workaround ++ vvtype out = (vvtype){0, 0}; ++ out = vec_insert((signed long long)v, out, 0); ++ out = vec_insert((signed long long)v, out, 1); ++ return out; ++ } ++}; ++ ++// 3. The long redirection ++// Place this INSIDE the defined(__clang__) block. ++// GCC usually treats int64_t and long as the same for templates, ++// but Clang 15 often sees them as distinct types requiring separate specializations. ++ ++#if defined(__powerpc64__) ++template ++struct vec_splats_dispatch::value && ++ !std::is_same::value>::type> ++ : vec_splats_dispatch {}; ++#endif ++ ++//template <> ++//struct vec_splats_dispatch : vec_splats_dispatch {}; ++ ++#endif ++ ++/* + #if defined(__clang__) && !defined(vec_splats) + C10_ALWAYS_INLINE vint64 vec_splats(const int64_t& a) { +- return vec_splats(a); ++ // Create a temporary array with the repeated value ++ __at_align__ double tmp[2]; // 2 elements for 128-bit vector ++ // Copy same bits of int64_t into both slots ++ std::memcpy(&tmp[0], &a, sizeof(a)); ++ std::memcpy(&tmp[1], &a, sizeof(a)); ++ // Load into vector register ++ return (vint64)vec_vsx_ld(0, tmp); ++ //return {{signed long long}a, {signed long long}a}; //vec_splats(a); + } + #endif ++*/ + // Vectorized min/max which return a if any operand is nan + template + C10_ALWAYS_INLINE T vec_min_nan(const T& a, const T& b) { +@@ -236,16 +287,47 @@ C10_VSX_VEC_NAN_PROPAG(vec_max_nan2, vfloat64, vbool64, vec_max) + return Vectorized{ret_0, ret_1}; \ + } + ++template ++struct canonical_scalar { ++ using type = T; ++}; ++ ++template <> ++struct canonical_scalar { ++ using type = int64_t; ++}; ++ ++template <> ++struct canonical_scalar { ++ using type = uint64_t; ++}; ++ ++#define DEFINE_MEMBER_OP_AND_ONE(op, op_type, func) \ ++ Vectorized::type> \ ++ C10_ALWAYS_INLINE op( \ ++ const Vectorized::type>& other) \ ++ const { \ ++ using scalar_t = typename canonical_scalar::type; \ ++ using vvtype = typename Vectorized::vec_internal_type; \ ++ const vvtype v_one = \ ++ vec_splats_dispatch::apply(static_cast(1)); \ ++ vvtype ret0 = (vvtype)func(_vec0, other._vec0); \ ++ vvtype ret1 = (vvtype)func(_vec1, other._vec1); \ ++ return Vectorized{vec_and(ret0, v_one), \ ++ vec_and(ret1, v_one)}; \ ++ } ++ ++/* + #define DEFINE_MEMBER_OP_AND_ONE(op, op_type, func) \ + Vectorized C10_ALWAYS_INLINE op(const Vectorized& other) \ + const { \ + using vvtype = Vectorized::vec_internal_type; \ +- const vvtype v_one = vec_splats(static_cast(1.0)); \ ++ const vvtype v_one = vec_splats_dispatch::apply(static_cast(1)); \ + vvtype ret0 = (vvtype)func(_vec0, other._vec0); \ + vvtype ret1 = (vvtype)func(_vec1, other._vec1); \ + return Vectorized{vec_and(ret0, v_one), vec_and(ret1, v_one)}; \ + } +- ++*/ + #define DEFINE_CLAMP_FUNCS(operand_type) \ + template <> \ + Vectorized C10_ALWAYS_INLINE clamp( \