Skip to content

Added build script for ml-commoms 3.5.0.0 #7969

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
prachi-gaonkar wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Added build script for ml-commoms 3.5.0.0 #7969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions o/opensearch-project-ml-commons/build_info.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
{
"maintainer": "Prachi.Gaonkar@ibm.com",
"package_name": "opensearch-project-ml-commons",
"github_url": "https://github.com/opensearch-project/ml-commons",
"version": "3.5.0.0",
"required_versions":{"Releases": ["*"], "Tags": ["*"]},
"default_branch": "main",
"package_dir": "o/opensearch-project-ml-commons",
"build_script": "ml-commons_3.5.0.0_ubi9.6.sh",
"validate_build_script": true,
"use_non_root_user": true,
"docker_build": false,
"3.5.0.0": {
"build_script": "ml-commons_3.5.0.0_ubi9.6.sh"
},
"3.3.0.0": {
"build_script": "ml-commons_3.3.0.0_ubi9.6.sh"
},
"2.19.2.0": {
"build_script": "ml-commons_2.19.2.0_ubi9.5.sh"
},
"*.*.*.*": {
"build_script": "ml-commons_3.5.0.0_ubi9.6.sh"
}
}
133 changes: 133 additions & 0 deletions o/opensearch-project-ml-commons/djl_v0.33.0_updated.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
diff --git a/engines/pytorch/pytorch-jni/build.gradle.kts b/engines/pytorch/pytorch-jni/build.gradle.kts
index c2c0c0c84..72c08b62a 100644
--- a/engines/pytorch/pytorch-jni/build.gradle.kts
+++ b/engines/pytorch/pytorch-jni/build.gradle.kts
@@ -29,33 +29,47 @@ tasks {
val hasJni = project.hasProperty("jni")

doFirst {
- val url = "https://publish.djl.ai/pytorch/$ptVersion/jnilib/${libs.versions.djl.get()}"
- val files = listOf(
- "linux-x86_64/cpu/libdjl_torch.so",
- "linux-x86_64/cpu-precxx11/libdjl_torch.so",
- "linux-aarch64/cpu-precxx11/libdjl_torch.so",
- "osx-aarch64/cpu/libdjl_torch.dylib",
- "win-x86_64/cpu/djl_torch.dll"
- ) + when {
- ptVersion.matches(Regex("2.[4-5].\\d")) -> listOf(
- "linux-x86_64/cu124/libdjl_torch.so",
- "linux-x86_64/cu124-precxx11/libdjl_torch.so",
- "win-x86_64/cu124/djl_torch.dll"
- )
-
- ptVersion.matches(Regex("2.[1-3].\\d")) -> listOf(
- "linux-x86_64/cu121/libdjl_torch.so",
- "linux-x86_64/cu121-precxx11/libdjl_torch.so",
- "win-x86_64/cu121/djl_torch.dll",
- )
-
- ptVersion.startsWith("1.13.") -> listOf(
- "linux-x86_64/cu117/libdjl_torch.so",
- "win-x86_64/cu117/djl_torch.dll",
- )
-
- else -> throw GradleException("Unsupported version: $ptVersion.")
- }
+ val url = "https://publish.djl.ai/pytorch/$ptVersion/jnilib/${libs.versions.djl.get()}"
+ val arch = System.getProperty("os.arch")
+
+ val files = when (arch) {
+ "ppc64le" -> listOf(
+ "linux-ppc64le/cpu/libdjl_torch.so"
+ )
+
+ "x86_64", "amd64" -> listOf(
+ "linux-x86_64/cpu/libdjl_torch.so",
+ "linux-x86_64/cpu-precxx11/libdjl_torch.so"
+ ) + when {
+ ptVersion.matches(Regex("2.[4-9].\\d")) -> listOf(
+ "linux-x86_64/cu124/libdjl_torch.so",
+ "linux-x86_64/cu124-precxx11/libdjl_torch.so",
+ "win-x86_64/cu124/djl_torch.dll"
+ )
+
+ ptVersion.matches(Regex("2.[1-3].\\d")) -> listOf(
+ "linux-x86_64/cu121/libdjl_torch.so",
+ "linux-x86_64/cu121-precxx11/libdjl_torch.so",
+ "win-x86_64/cu121/djl_torch.dll"
+ )
+
+ ptVersion.startsWith("1.13.") -> listOf(
+ "linux-x86_64/cu117/libdjl_torch.so",
+ "win-x86_64/cu117/djl_torch.dll"
+ )
+
+ else -> emptyList()
+ }
+
+ "aarch64" -> listOf(
+ "linux-aarch64/cpu-precxx11/libdjl_torch.so",
+ "osx-aarch64/cpu/libdjl_torch.dylib"
+ )
+
+ else -> throw GradleException("Unsupported arch: $arch")
+
+ }
+
val jnilibDir = dir / "jnilib" / libs.versions.djl.get()
for (entry in files) {
val file = jnilibDir / entry
diff --git a/engines/pytorch/pytorch-native/build.gradle.kts b/engines/pytorch/pytorch-native/build.gradle.kts
index e5f002961..7d877e93a 100644
--- a/engines/pytorch/pytorch-native/build.gradle.kts
+++ b/engines/pytorch/pytorch-native/build.gradle.kts
@@ -33,7 +33,7 @@ fun downloadBuild(
isPrecxx11: Boolean = false,
isAarch64: Boolean = false
) {
- val arch = if (isAarch64) "aarch64" else "x86_64"
+ val arch = if (isAarch64) "aarch64" else "ppc64le"
execOperations.exec {
workingDir = project.projectDir
if (os == "win")
@@ -92,6 +92,7 @@ fun prepareNativeLib(execOperations: ExecOperations, binaryRoot: String, ver: St
} else {
// @formatter:off
val files = mapOf(
+ "cpu/libtorch-cxx11-abi-shared-with-deps-$ver%2Bcpu.zip" to "cpu/linux-ppc64le",
"cpu/libtorch-cxx11-abi-shared-with-deps-$ver%2Bcpu.zip" to "cpu/linux-x86_64",
"cpu/libtorch-macos-arm64-$ver.zip" to "cpu/osx-aarch64",
"cpu/libtorch-win-shared-with-deps-$ver%2Bcpu.zip" to "cpu/win-x86_64",
@@ -256,6 +257,7 @@ tasks {

(binaryRoot / "files.txt").text = buildString {
val uploadDirs = listOf(
+ binaryRoot / "cpu/linux-ppc64le/native/lib/",
binaryRoot / "cpu/linux-x86_64/native/lib/",
binaryRoot / "cpu/osx-aarch64/native/lib/",
binaryRoot / "cpu/win-x86_64/native/lib/",
diff --git a/extensions/tokenizers/build.gradle.kts b/extensions/tokenizers/build.gradle.kts
index 8108d174d..05adfb973 100644
--- a/extensions/tokenizers/build.gradle.kts
+++ b/extensions/tokenizers/build.gradle.kts
@@ -52,6 +52,7 @@ tasks {
"win-x86_64/cpu/libgcc_s_seh-1.dll" to "extra/win-x86_64/libgcc_s_seh-1.dll",
"win-x86_64/cpu/libstdc%2B%2B-6.dll" to "extra/win-x86_64/libstdc%2B%2B-6.dll",
"win-x86_64/cpu/tokenizers.dll" to "$tokenizers/jnilib/$djl",
+ "linux-ppc64le/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl",
"linux-x86_64/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl",
"linux-aarch64/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl",
"osx-aarch64/cpu/libtokenizers.dylib" to "$tokenizers/jnilib/$djl"
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index fcb16e641..d6db2877b 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -3,7 +3,7 @@ format.version = "1.1"

[versions]
djl = "0.33.0"
-pytorch = "2.5.1"
+pytorch = "2.9.1"
tensorflow = "2.16.1"
tensorflowCore = "1.0.0-rc.1"
mxnet = "1.9.1"
4 changes: 2 additions & 2 deletions o/opensearch-project-ml-commons/ml-commons_3.3.0.0_ubi9.6.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ wget https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/master/o
git clone https://github.com/deepjavalibrary/djl
cd djl/
git checkout $DJL_VERSION
git apply $BUILD_HOME/djl_$DJL_VERSION.patch
git apply $BUILD_HOME/djl_$DJL_VERSION_updated.patch
wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip
unzip libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}+cpu.zip -d $BUILD_HOME/djl/engines/pytorch/pytorch-native
rm -rf libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}+cpu.zip
Expand Down Expand Up @@ -249,7 +249,7 @@ git apply ${SCRIPT_PATH}/ml-commons_$SCRIPT_PACKAGE_VERSION.patch
# Build
# --------
ret=0
./gradlew build -x test -x integTest -Dbuild.snapshot=false|| ret=$?
./gradlew build -x test -x integTest || ret=$?
if [ $ret -ne 0 ]; then
set +ex
echo "------------------ ${PACKAGE_NAME}: Build Failed ------------------"
Expand Down
Loading
Loading