adding machine learning computational services

.bumpversion-pytorch.cfg

@@ -0,0 +1,13 @@
+[bumpversion]
+current_version = 1.1.0
+commit = False
+message = service version: {current_version} → {new_version}
+tag = False
+
+[bumpversion:file:.osparc/osparc-python-runner-pytorch/metadata.yml]
+search = version: {current_version}
+replace = version: {new_version}
+
+[bumpversion:file:Makefile]
+search = TAG_PYTORCH={current_version}
+replace = TAG_PYTORCH={new_version}

.bumpversion-tensorflow.cfg

@@ -0,0 +1,13 @@
+[bumpversion]
+current_version = 1.1.0
+commit = False
+message = service version: {current_version} → {new_version}
+tag = False
+
+[bumpversion:file:.osparc/osparc-python-runner-tensorflow/metadata.yml]
+search = version: {current_version}
+replace = version: {new_version}
+
+[bumpversion:file:Makefile]
+search = TAG_TENSORFLOW={current_version}
+replace = TAG_TENSORFLOW={new_version}

.github/workflows/check-image.yml

@@ -0,0 +1,37 @@
+name: Build and check image
+
+on:
+  # Run the workflow for pushes to the main branch
+  push:
+    branches:
+      - master
+
+  # Run the workflow for pull requests (from forks or within the repo)
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+jobs:
+  verify-image-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo content
+        uses: actions/checkout@v2
+      - name: ooil version
+        uses: docker://itisfoundation/ci-service-integration-library:v2.0.11
+        with:
+          args: ooil --version
+      - name: Assemble docker compose spec
+        uses: docker://itisfoundation/ci-service-integration-library:v2.0.11
+        with:
+          args: ooil compose
+      - name: Build all images if multiple
+        uses: docker://itisfoundation/ci-service-integration-library:v2.0.11
+        with:
+          args: docker compose build
+      - name: test Tensorflow
+        run: make run-tensorflow-local
+      - name: test PyTorch
+        run: make run-pytorch-local

.gitignore

@@ -0,0 +1,8 @@
+# not interested in storing this file
+docker-compose.yml
+
+
+*.ignore.*
+
+.venv/
+docker-compose.yml

.osparc/common/docker-compose.overwrite.yml

@@ -0,0 +1,5 @@
+services:
+  common:
+    build:
+      context: ./common
+      dockerfile: Dockerfile

.osparc/common/metadata.yml

@@ -0,0 +1,14 @@
+name: "Base runner-ml image"
+thumbnail:
+description: "used only for building not to be published"
+key: simcore/services/comp/common
+version: 0.0.0
+integration-version: 2.0.0
+type: dynamic
+authors:
+  - name: "Andrei Neagu"
+    email: "neagu@itis.swiss"
+    affiliation: "IT'IS Foundation"
+contact: "neagu@itis.swiss"
+inputs: {}
+outputs: {}

.osparc/common/runtime.yml

@@ -0,0 +1 @@
+settings: []

.osparc/osparc-python-runner-pytorch/docker-compose.overwrite.yml

@@ -0,0 +1,7 @@
+services:
+  osparc-python-runner-pytorch:
+    depends_on: 
+      - common
+    build:
+      context: ./osparc-python-runner-pytorch
+      dockerfile: Dockerfile

.osparc/osparc-python-runner-pytorch/metadata.yml

@@ -0,0 +1,68 @@
+name: oSparc Python Runner PyTorch
+key: simcore/services/comp/osparc-python-runner-pytorch
+type: computational
+integration-version: 2.0.0
+version: 1.1.0
+description: https://raw.githubusercontent.com/ZurichMedTech/s4l-assets/refs/heads/main/app/full/services/simcore_services_comp_osparc-python-runner-pytorch.md
+icon: https://raw.githubusercontent.com/ZurichMedTech/s4l-assets/main/app/icons/s4l/simcore_services_comp_osparc-python-runner-pytorch.png
+thumbnail: https://raw.githubusercontent.com/ZurichMedTech/s4l-assets/main/app/thumbnails/s4l/simcore_services_comp_osparc-python-runner-pytorch.png
+contact: guidon@itis.swiss
+authors:
+  - name: Manuel Guidon
+    email: guidon@itis.swiss
+    affiliation: IT'IS Foundation
+inputs:
+  input_1:
+    displayOrder: 1
+    label: Input data
+    description: Any code, requirements or data file
+    type: data:*/*
+  input_2:
+    displayOrder: 2
+    label: Additional input data - optional
+    description: Any additional input data file
+    type: data:*/*
+  input_3:
+    displayOrder: 3
+    label: Additional input data - optional
+    description: Any additional input data file
+    type: data:*/*
+  input_4:
+    displayOrder: 4
+    label: Additional input data - optional
+    description: Any additional input data file
+    type: data:*/*
+  input_5:
+    displayOrder: 5
+    label: Additional input data - optional
+    description: Any additional input data file
+    type: data:*/*
+outputs:
+  output_1:
+    displayOrder: 1
+    label: Output data
+    description: The data produced by the script and saved under OUTPUT_FOLDER/output_1 as output_1.zip
+    type: data:*/*
+    fileToKeyMap:
+      output_1.zip: output_1
+  output_2:
+    displayOrder: 2
+    label: Output data
+    description: The data produced by the script and saved under OUTPUT_FOLDER/output_2 as output_2.zip
+    type: data:*/*
+    fileToKeyMap:
+      output_2.zip: output_2
+  output_3:
+    displayOrder: 3
+    label: Output data
+    description: The data produced by the script and saved under OUTPUT_FOLDER/output_3 as output_3.zip
+    type: data:*/*
+    fileToKeyMap:
+      output_3.zip: output_3
+  output_4:
+    displayOrder: 4
+    label: Output data
+    description: The data produced by the script and saved under OUTPUT_FOLDER/output_4 as output_4.zip
+    type: data:*/*
+    fileToKeyMap:
+      output_4.zip: output_4

.osparc/osparc-python-runner-pytorch/runtime.yml

@@ -0,0 +1,14 @@
+settings:
+  - name: Resources
+    type: Resources
+    value:
+      Limits:
+        NanoCPUs: 0
+        MemoryBytes: 0
+      Reservations:
+        NanoCPUs: 0
+        MemoryBytes: 0
+        GenericResources:
+          - DiscreteResourceSpec:
+              Kind: VRAM
+              Value: 1

.osparc/osparc-python-runner-tensorflow/docker-compose.overwrite.yml

@@ -0,0 +1,7 @@
+services:
+  osparc-python-runner-tensorflow:
+    depends_on: 
+      - common
+    build:
+      context: ./osparc-python-runner-tensorflow
+      dockerfile: Dockerfile

.osparc/osparc-python-runner-tensorflow/metadata.yml

@@ -0,0 +1,68 @@
+name: oSparc Python Runner Tensorflow
+key: simcore/services/comp/osparc-python-runner-tensorflow
+type: computational
+integration-version: 2.0.0
+version: 1.1.0
+description: https://raw.githubusercontent.com/ZurichMedTech/s4l-assets/refs/heads/main/app/full/services/simcore_services_comp_osparc-python-runner-tensorflow.md
+icon: https://raw.githubusercontent.com/ZurichMedTech/s4l-assets/main/app/icons/s4l/simcore_services_comp_osparc-python-runner-tensorflow.png
+thumbnail: https://raw.githubusercontent.com/ZurichMedTech/s4l-assets/main/app/thumbnails/s4l/simcore_services_comp_osparc-python-runner-tensorflow.png
+contact: guidon@itis.swiss
+authors:
+  - name: Manuel Guidon
+    email: guidon@itis.swiss
+    affiliation: IT'IS Foundation
+inputs:
+  input_1:
+    displayOrder: 1
+    label: Input data
+    description: Any code, requirements or data file
+    type: data:*/*
+  input_2:
+    displayOrder: 2
+    label: Additional input data - optional
+    description: Any additional input data file
+    type: data:*/*
+  input_3:
+    displayOrder: 3
+    label: Additional input data - optional
+    description: Any additional input data file
+    type: data:*/*
+  input_4:
+    displayOrder: 4
+    label: Additional input data - optional
+    description: Any additional input data file
+    type: data:*/*
+  input_5:
+    displayOrder: 5
+    label: Additional input data - optional
+    description: Any additional input data file
+    type: data:*/*
+outputs:
+  output_1:
+    displayOrder: 1
+    label: Output data
+    description: The data produced by the script and saved under OUTPUT_FOLDER/output_1 as output_1.zip
+    type: data:*/*
+    fileToKeyMap:
+      output_1.zip: output_1
+  output_2:
+    displayOrder: 2
+    label: Output data
+    description: The data produced by the script and saved under OUTPUT_FOLDER/output_2 as output_2.zip
+    type: data:*/*
+    fileToKeyMap:
+      output_2.zip: output_2
+  output_3:
+    displayOrder: 3
+    label: Output data
+    description: The data produced by the script and saved under OUTPUT_FOLDER/output_3 as output_3.zip
+    type: data:*/*
+    fileToKeyMap:
+      output_3.zip: output_3
+  output_4:
+    displayOrder: 4
+    label: Output data
+    description: The data produced by the script and saved under OUTPUT_FOLDER/output_4 as output_4.zip
+    type: data:*/*
+    fileToKeyMap:
+      output_4.zip: output_4

.osparc/osparc-python-runner-tensorflow/runtime.yml

@@ -0,0 +1,14 @@
+settings:
+  - name: Resources
+    type: Resources
+    value:
+      Limits:
+        NanoCPUs: 0
+        MemoryBytes: 0
+      Reservations:
+        NanoCPUs: 0
+        MemoryBytes: 0
+        GenericResources:
+          - DiscreteResourceSpec:
+              Kind: VRAM
+              Value: 1

Makefile

@@ -0,0 +1,102 @@
+# minimalistic utility to test and develop locally
+
+SHELL = /bin/sh
+.DEFAULT_GOAL := help
+
+export IMAGE_PYTORCH=osparc-python-runner-pytorch
+export IMAGE_TENSORFLOW=osparc-python-runner-tensorflow
+export TAG_PYTORCH=1.1.0
+export TAG_TENSORFLOW=1.1.0
+
+# PYTHON ENVIRON ---------------------------------------------------------------------------------------
+.PHONY: devenv
+.venv:
+	@python3 --version
+	python3 -m venv $@
+	# upgrading package managers
+	$@/bin/pip install --upgrade uv
+
+devenv: .venv  ## create a python virtual environment with tools to dev, run and tests cookie-cutter
+	# installing extra tools
+	@$</bin/uv pip install wheel setuptools
+	# your dev environment contains
+	@$</bin/uv pip list
+	@echo "To activate the virtual environment, run 'source $</bin/activate'"
+
+# Builds new service version ----------------------------------------------------------------------------
+define _bumpversion
+	# upgrades as $(subst $(1),,$@) version, commits and tags
+	@docker run -it --rm -v $(PWD):/ml-runner \
+		-u $(shell id -u):$(shell id -g) \
+		itisfoundation/ci-service-integration-library:v2.0.11 \
+		sh -c "cd /ml-runner && bump2version --verbose --list --config-file $(1) $(subst $(2),,$@)"
+endef
+
+.PHONY: version-tensorflow-patch version-tensorflow-minor version-tensorflow-major
+version-tensorflow-patch version-tensorflow-minor version-tensorflow-major: .bumpversion-tensorflow.cfg ## increases tensroflow service's version
+	@make compose-spec
+	@$(call _bumpversion,$<,version-tensorflow-)
+	@make compose-spec
+
+.PHONY: version-pytorch-patch version-pytorch-minor version-pytorch-major
+version-pytorch-patch version-pytorch-minor version-pytorch-major: .bumpversion-pytorch.cfg ## increases pytorchservice's version
+	@make compose-spec
+	@$(call _bumpversion,$<,version-pytorch-)
+	@make compose-spec
+
+
+define _create_run_script
+	@docker run -it --rm -v $(PWD):/ml-runner \
+		-u $(shell id -u):$(shell id -g) \
+		itisfoundation/ci-service-integration-library:v2.0.11 \
+		sh -c "cd /ml-runner && \
+			ooil run-creator \
+				--runscript $(1)/service.cli/run \
+				--metadata .osparc/$(1)/metadata.yml"
+endef
+
+.PHONY: create-run-script
+create-run-script: ## assembles run scrips for pytorch and tensorflow
+	@$(call _create_run_script,osparc-python-runner-pytorch)
+	@$(call _create_run_script,osparc-python-runner-tensorflow)
+	# TODO: add check to make sure git state is not dirty in CI so it fails
+
+.PHONY: compose-spec
+compose-spec: ## runs ooil to assemble the docker-compose.yml file
+	@docker run -it --rm -v $(PWD):/ml-runner \
+		-u $(shell id -u):$(shell id -g) \
+		itisfoundation/ci-service-integration-library:v2.0.11 \
+		sh -c "cd /ml-runner && ooil compose"
+
+build: | compose-spec	## build docker image
+	docker compose build
+
+# To test built service locally -------------------------------------------------------------------------
+.PHONY: run-pytorch-local
+run-pytorch-local: ## runs pytorch image with local configuration
+	IMAGE_TO_RUN=${IMAGE_PYTORCH} \
+	TAG_TO_RUN=${TAG_PYTORCH} \
+	VALIDATION_DIR=validation-pytorch \
+	docker compose --file docker-compose-local.yml up --abort-on-container-exit --exit-code-from runner-ml
+
+.PHONY: run-tensorflow-local
+run-tensorflow-local: ## runs tensorflow image with local configuration
+	IMAGE_TO_RUN=${IMAGE_TENSORFLOW} \
+	TAG_TO_RUN=${TAG_TENSORFLOW} \
+	VALIDATION_DIR=validation-tensorflow \
+	docker compose --file docker-compose-local.yml up --abort-on-container-exit --exit-code-from runner-ml
+
+.PHONY: publish-local
+publish-local: ## push to local throw away registry to test integration
+	docker tag simcore/services/comp/${IMAGE_PYTORCH}:${TAG_PYTORCH} registry:5000/simcore/services/comp/${IMAGE_PYTORCH}:${TAG_PYTORCH}
+	docker push registry:5000/simcore/services/comp/${IMAGE_PYTORCH}:${TAG_PYTORCH}
+	docker tag simcore/services/comp/${IMAGE_TENSORFLOW}:${TAG_TENSORFLOW} registry:5000/simcore/services/comp/${IMAGE_TENSORFLOW}:${TAG_TENSORFLOW}
+	docker push registry:5000/simcore/services/comp/${IMAGE_TENSORFLOW}:${TAG_TENSORFLOW}
+	@curl registry:5000/v2/_catalog | jq
+
+.PHONY: help
+help: ## this colorful help
+	@echo "Recipes for '$(notdir $(CURDIR))':"
+	@echo ""
+	@awk 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+	@echo ""