Skip to content

Commit 92bbeca

Browse files
prachi-gaonkarprachi-gaonkar
committed
Added build script for ml-commoms 3.5.0.0
1 parent 746a0bb commit 92bbeca

5 files changed

Lines changed: 685 additions & 2 deletions

File tree

‎o/opensearch-project-ml-commons/build_info.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"maintainer": "Prachi.Gaonkar@ibm.com",
3+
"package_name": "opensearch-project-ml-commons",
4+
"github_url": "https://github.com/opensearch-project/ml-commons",
5+
"version": "3.5.0.0",
6+
"required_versions":{"Releases": ["*"], "Tags": ["*"]},
7+
"default_branch": "main",
8+
"package_dir": "o/opensearch-project-ml-commons",
9+
"build_script": "ml-commons_3.5.0.0_ubi9.6.sh",
10+
"validate_build_script": true,
11+
"use_non_root_user": true,
12+
"docker_build": false,
13+
"3.5.0.0": {
14+
"build_script": "ml-commons_3.5.0.0_ubi9.6.sh"
15+
},
16+
"3.3.0.0": {
17+
"build_script": "ml-commons_3.3.0.0_ubi9.6.sh"
18+
},
19+
"2.19.2.0": {
20+
"build_script": "ml-commons_2.19.2.0_ubi9.5.sh"
21+
},
22+
"*.*.*.*": {
23+
"build_script": "ml-commons_3.5.0.0_ubi9.6.sh"
24+
}
25+
}

‎o/opensearch-project-ml-commons/djl_v0.33.0_updated.patch‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
diff --git a/engines/pytorch/pytorch-jni/build.gradle.kts b/engines/pytorch/pytorch-jni/build.gradle.kts
2+
index c2c0c0c84..72c08b62a 100644
3+
--- a/engines/pytorch/pytorch-jni/build.gradle.kts
4+
+++ b/engines/pytorch/pytorch-jni/build.gradle.kts
5+
@@ -29,33 +29,47 @@ tasks {
6+
val hasJni = project.hasProperty("jni")
7+
8+
doFirst {
9+
- val url = "https://publish.djl.ai/pytorch/$ptVersion/jnilib/${libs.versions.djl.get()}"
10+
- val files = listOf(
11+
- "linux-x86_64/cpu/libdjl_torch.so",
12+
- "linux-x86_64/cpu-precxx11/libdjl_torch.so",
13+
- "linux-aarch64/cpu-precxx11/libdjl_torch.so",
14+
- "osx-aarch64/cpu/libdjl_torch.dylib",
15+
- "win-x86_64/cpu/djl_torch.dll"
16+
- ) + when {
17+
- ptVersion.matches(Regex("2.[4-5].\\d")) -> listOf(
18+
- "linux-x86_64/cu124/libdjl_torch.so",
19+
- "linux-x86_64/cu124-precxx11/libdjl_torch.so",
20+
- "win-x86_64/cu124/djl_torch.dll"
21+
- )
22+
-
23+
- ptVersion.matches(Regex("2.[1-3].\\d")) -> listOf(
24+
- "linux-x86_64/cu121/libdjl_torch.so",
25+
- "linux-x86_64/cu121-precxx11/libdjl_torch.so",
26+
- "win-x86_64/cu121/djl_torch.dll",
27+
- )
28+
-
29+
- ptVersion.startsWith("1.13.") -> listOf(
30+
- "linux-x86_64/cu117/libdjl_torch.so",
31+
- "win-x86_64/cu117/djl_torch.dll",
32+
- )
33+
-
34+
- else -> throw GradleException("Unsupported version: $ptVersion.")
35+
- }
36+
+ val url = "https://publish.djl.ai/pytorch/$ptVersion/jnilib/${libs.versions.djl.get()}"
37+
+ val arch = System.getProperty("os.arch")
38+
+
39+
+ val files = when (arch) {
40+
+ "ppc64le" -> listOf(
41+
+ "linux-ppc64le/cpu/libdjl_torch.so"
42+
+ )
43+
+
44+
+ "x86_64", "amd64" -> listOf(
45+
+ "linux-x86_64/cpu/libdjl_torch.so",
46+
+ "linux-x86_64/cpu-precxx11/libdjl_torch.so"
47+
+ ) + when {
48+
+ ptVersion.matches(Regex("2.[4-9].\\d")) -> listOf(
49+
+ "linux-x86_64/cu124/libdjl_torch.so",
50+
+ "linux-x86_64/cu124-precxx11/libdjl_torch.so",
51+
+ "win-x86_64/cu124/djl_torch.dll"
52+
+ )
53+
+
54+
+ ptVersion.matches(Regex("2.[1-3].\\d")) -> listOf(
55+
+ "linux-x86_64/cu121/libdjl_torch.so",
56+
+ "linux-x86_64/cu121-precxx11/libdjl_torch.so",
57+
+ "win-x86_64/cu121/djl_torch.dll"
58+
+ )
59+
+
60+
+ ptVersion.startsWith("1.13.") -> listOf(
61+
+ "linux-x86_64/cu117/libdjl_torch.so",
62+
+ "win-x86_64/cu117/djl_torch.dll"
63+
+ )
64+
+
65+
+ else -> emptyList()
66+
+ }
67+
+
68+
+ "aarch64" -> listOf(
69+
+ "linux-aarch64/cpu-precxx11/libdjl_torch.so",
70+
+ "osx-aarch64/cpu/libdjl_torch.dylib"
71+
+ )
72+
+
73+
+ else -> throw GradleException("Unsupported arch: $arch")
74+
+
75+
+ }
76+
+
77+
val jnilibDir = dir / "jnilib" / libs.versions.djl.get()
78+
for (entry in files) {
79+
val file = jnilibDir / entry
80+
diff --git a/engines/pytorch/pytorch-native/build.gradle.kts b/engines/pytorch/pytorch-native/build.gradle.kts
81+
index e5f002961..7d877e93a 100644
82+
--- a/engines/pytorch/pytorch-native/build.gradle.kts
83+
+++ b/engines/pytorch/pytorch-native/build.gradle.kts
84+
@@ -33,7 +33,7 @@ fun downloadBuild(
85+
isPrecxx11: Boolean = false,
86+
isAarch64: Boolean = false
87+
) {
88+
- val arch = if (isAarch64) "aarch64" else "x86_64"
89+
+ val arch = if (isAarch64) "aarch64" else "ppc64le"
90+
execOperations.exec {
91+
workingDir = project.projectDir
92+
if (os == "win")
93+
@@ -92,6 +92,7 @@ fun prepareNativeLib(execOperations: ExecOperations, binaryRoot: String, ver: St
94+
} else {
95+
// @formatter:off
96+
val files = mapOf(
97+
+ "cpu/libtorch-cxx11-abi-shared-with-deps-$ver%2Bcpu.zip" to "cpu/linux-ppc64le",
98+
"cpu/libtorch-cxx11-abi-shared-with-deps-$ver%2Bcpu.zip" to "cpu/linux-x86_64",
99+
"cpu/libtorch-macos-arm64-$ver.zip" to "cpu/osx-aarch64",
100+
"cpu/libtorch-win-shared-with-deps-$ver%2Bcpu.zip" to "cpu/win-x86_64",
101+
@@ -256,6 +257,7 @@ tasks {
102+
103+
(binaryRoot / "files.txt").text = buildString {
104+
val uploadDirs = listOf(
105+
+ binaryRoot / "cpu/linux-ppc64le/native/lib/",
106+
binaryRoot / "cpu/linux-x86_64/native/lib/",
107+
binaryRoot / "cpu/osx-aarch64/native/lib/",
108+
binaryRoot / "cpu/win-x86_64/native/lib/",
109+
diff --git a/extensions/tokenizers/build.gradle.kts b/extensions/tokenizers/build.gradle.kts
110+
index 8108d174d..05adfb973 100644
111+
--- a/extensions/tokenizers/build.gradle.kts
112+
+++ b/extensions/tokenizers/build.gradle.kts
113+
@@ -52,6 +52,7 @@ tasks {
114+
"win-x86_64/cpu/libgcc_s_seh-1.dll" to "extra/win-x86_64/libgcc_s_seh-1.dll",
115+
"win-x86_64/cpu/libstdc%2B%2B-6.dll" to "extra/win-x86_64/libstdc%2B%2B-6.dll",
116+
"win-x86_64/cpu/tokenizers.dll" to "$tokenizers/jnilib/$djl",
117+
+ "linux-ppc64le/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl",
118+
"linux-x86_64/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl",
119+
"linux-aarch64/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl",
120+
"osx-aarch64/cpu/libtokenizers.dylib" to "$tokenizers/jnilib/$djl"
121+
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
122+
index fcb16e641..d6db2877b 100644
123+
--- a/gradle/libs.versions.toml
124+
+++ b/gradle/libs.versions.toml
125+
@@ -3,7 +3,7 @@ format.version = "1.1"
126+
127+
[versions]
128+
djl = "0.33.0"
129+
-pytorch = "2.5.1"
130+
+pytorch = "2.9.1"
131+
tensorflow = "2.16.1"
132+
tensorflowCore = "1.0.0-rc.1"
133+
mxnet = "1.9.1"

‎o/opensearch-project-ml-commons/ml-commons_3.3.0.0_ubi9.6.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ wget https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/master/o
166166
git clone https://github.com/deepjavalibrary/djl
167167
cd djl/
168168
git checkout $DJL_VERSION
169-
git apply $BUILD_HOME/djl_$DJL_VERSION.patch
169+
git apply $BUILD_HOME/djl_$DJL_VERSION_updated.patch
170170
wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip
171171
unzip libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}+cpu.zip -d $BUILD_HOME/djl/engines/pytorch/pytorch-native
172172
rm -rf libtorch-cxx11-abi-shared-with-deps-${PYTORCH_VERSION}+cpu.zip
@@ -249,7 +249,7 @@ git apply ${SCRIPT_PATH}/ml-commons_$SCRIPT_PACKAGE_VERSION.patch
249249
# Build
250250
# --------
251251
ret=0
252-
./gradlew build -x test -x integTest -Dbuild.snapshot=false|| ret=$?
252+
./gradlew build -x test -x integTest || ret=$?
253253
if [ $ret -ne 0 ]; then
254254
set +ex
255255
echo "------------------ ${PACKAGE_NAME}: Build Failed ------------------"

0 commit comments

Comments
 (0)