From 366bc197412ca32d340dc8271b8220be31e9ed1f Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Thu, 23 Jul 2026 10:07:11 +0800 Subject: [PATCH 1/3] docs: add reusable pipeline components guide --- .../assets/pipelines/verify-kfp-features.py | 110 +++++++++ docs/en/training_guides/index.mdx | 2 + .../kfp-execution-and-storage.mdx | 167 ++++++++++++++ .../reusable-pipeline-components.mdx | 209 ++++++++++++++++++ 4 files changed, 488 insertions(+) create mode 100644 docs/en/training_guides/assets/pipelines/verify-kfp-features.py create mode 100644 docs/en/training_guides/kfp-execution-and-storage.mdx create mode 100644 docs/en/training_guides/reusable-pipeline-components.mdx diff --git a/docs/en/training_guides/assets/pipelines/verify-kfp-features.py b/docs/en/training_guides/assets/pipelines/verify-kfp-features.py new file mode 100644 index 0000000..4af5580 --- /dev/null +++ b/docs/en/training_guides/assets/pipelines/verify-kfp-features.py @@ -0,0 +1,110 @@ +"""Verify KFP caching, ParallelFor, and persisted artifact passing.""" + +import os +import time + +from kfp import Client, compiler, dsl + + +@dsl.component(base_image="docker-mirrors.alauda.cn/library/python:3.12-slim") +def produce_dataset(seed: str, dataset: dsl.Output[dsl.Dataset]) -> None: + """Write a small dataset to the KFP artifact path.""" + from pathlib import Path + + output = Path(dataset.path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(f"seed={seed}\n", encoding="utf-8") + dataset.metadata["seed"] = seed + + +@dsl.component(base_image="docker-mirrors.alauda.cn/library/python:3.12-slim") +def verify_dataset(dataset: dsl.Input[dsl.Dataset], item: int, seed: str) -> str: + """Read the persisted artifact in one ParallelFor iteration.""" + from pathlib import Path + + content = Path(dataset.path).read_text(encoding="utf-8") + expected = f"seed={seed}\n" + if content != expected: + raise RuntimeError(f"artifact content {content!r} != {expected!r}") + return f"item={item}:{content.strip()}" + + +@dsl.pipeline(name="kfp-mechanisms-verification") +def mechanisms_pipeline(seed: str = "alauda-kfp-features") -> None: + """Produce one artifact and consume it in three parallel tasks.""" + producer = produce_dataset(seed=seed) + with dsl.ParallelFor(items=[1, 2, 3], parallelism=3) as item: + verify_dataset(dataset=producer.outputs["dataset"], item=item, seed=seed) + + +def task_state(task: object) -> str: + """Return a task state as a stable uppercase string.""" + value = getattr(task, "state", "") + return str(value).rsplit(".", 1)[-1].upper() + + +def run_once( + client: Client, package_path: str, experiment: str, suffix: str, seed: str +) -> object: + """Submit a cached run and wait for its terminal state.""" + result = client.create_run_from_pipeline_package( + pipeline_file=package_path, + arguments={"seed": seed}, + run_name=f"kfp-mechanisms-{suffix}-{int(time.time())}", + experiment_name=experiment, + namespace=os.environ["KFP_NAMESPACE"], + enable_caching=True, + ) + run = client.wait_for_run_completion(result.run_id, timeout=900, sleep_duration=5) + if str(run.state).upper() != "SUCCEEDED": + raise RuntimeError(f"run {result.run_id} ended in {run.state}") + print(f"run {suffix}: {result.run_id} state={run.state}", flush=True) + return client.get_run(result.run_id) + + +def main() -> None: + """Compile the pipeline, run it twice, and verify the three mechanisms.""" + package_path = "/tmp/kfp-mechanisms-verification.yaml" + compiler.Compiler().compile(mechanisms_pipeline, package_path=package_path) + client = Client(host=os.environ["KFP_ENDPOINT"], namespace=os.environ["KFP_NAMESPACE"]) + experiment = f"kfp-mechanisms-{int(time.time())}" + seed = f"alauda-kfp-features-{time.time_ns()}" + + first = run_once(client, package_path, experiment, "first", seed) + first_tasks = first.run_details.task_details or [] + for task in first_tasks: + print( + f"first task: name={task.display_name} state={task_state(task)} " + f"outputs={task.outputs or {}}", + flush=True, + ) + + parallel_tasks = [task for task in first_tasks if task.display_name == "verify-dataset"] + if len(parallel_tasks) != 3 or any(task_state(task) != "SUCCEEDED" for task in parallel_tasks): + raise RuntimeError( + f"ParallelFor expected 3 successful verify-dataset tasks, got {len(parallel_tasks)}" + ) + + producers = [task for task in first_tasks if task.display_name == "produce-dataset"] + if len(producers) != 1 or task_state(producers[0]) != "SUCCEEDED": + raise RuntimeError(f"expected one produce-dataset task, got {len(producers)}") + print( + "artifact persistence: producer uploaded the Dataset and all 3 consumers read it", + flush=True, + ) + + second = run_once(client, package_path, experiment, "cached", seed) + second_tasks = second.run_details.task_details or [] + for task in second_tasks: + print(f"cached task: name={task.display_name} state={task_state(task)}", flush=True) + skipped = [task for task in second_tasks if task_state(task) == "SKIPPED"] + if not skipped: + raise RuntimeError("second identical run did not report any cached (SKIPPED) task") + + print("PASS: ParallelFor created 3 tasks", flush=True) + print("PASS: Dataset artifact was persisted and consumed", flush=True) + print(f"PASS: cache reused {len(skipped)} task execution(s)", flush=True) + + +if __name__ == "__main__": + main() diff --git a/docs/en/training_guides/index.mdx b/docs/en/training_guides/index.mdx index 554e1bf..7840ec9 100644 --- a/docs/en/training_guides/index.mdx +++ b/docs/en/training_guides/index.mdx @@ -20,4 +20,6 @@ End-to-end recipes for fine-tuning and pretraining LLMs on Alauda AI. | Interactive exploration, custom scripts, VolcanoJob submission | Workbench Notebook | [Fine-tuning LLMs using Workbench](./fine-tuning-using-notebooks.mdx) | | Full-parameter SFT / pretraining on Ascend NPU | Workbench `PyTorch CANN` / `MindSpore CANN` | [Fine-tune and Pretrain on Ascend NPU](./fine-tune-and-pretrain-llms-on-ascend-npu.mdx) | | Track a KFP run's parameters and metrics in MLflow | KFP component + MLflow SDK | [Kubeflow Pipeline + MLflow Integration](../mlflow/how_to/pipelines-mlflow-integration.mdx) | +| Train tabular or time-series models with reusable AutoGluon assets | Managed KFP pipelines or composable components | [Use Reusable Kubeflow Pipeline Components](./reusable-pipeline-components.mdx) | +| Use KFP caching, parallel loops, and persistent typed artifacts | KFP 2.16.1 execution mechanisms | [Kubeflow Pipelines Execution and Storage Behavior](./kfp-execution-and-storage.mdx) | | Daily fine-tune → evaluate → compare loop with MLflow + TrustyAI | KFP Recurring Run + MLflow Model Registry + `LMEvalJob` | [Daily Fine-Tuning Pipeline with MLflow and TrustyAI](./fine-tuning-pipeline-with-mlflow-trustyai.mdx) | diff --git a/docs/en/training_guides/kfp-execution-and-storage.mdx b/docs/en/training_guides/kfp-execution-and-storage.mdx new file mode 100644 index 0000000..1baa77a --- /dev/null +++ b/docs/en/training_guides/kfp-execution-and-storage.mdx @@ -0,0 +1,167 @@ +--- +weight: 59 +--- + +# Kubeflow Pipelines Execution and Storage Behavior + +Kubeflow Pipelines 2.16.1 installed by the Data Science Pipelines Operator +(DSPO) supports task caching, `dsl.ParallelFor`, typed KFP artifacts, and +S3-compatible artifact persistence on x86 clusters. These mechanisms are KFP +features; DSPO configures the API server, workflow controller, cache server, +and artifact store that implement them. + +## Feature summary + +| Mechanism | DSPO on x86 | Behavior | +|---|---|---| +| Task caching | Supported | An identical task execution can be reused and is reported as `SKIPPED` in the repeated run. | +| `dsl.ParallelFor` | Supported | Each loop item becomes a separate task; `parallelism` limits concurrent iterations. | +| Typed artifacts | Supported | `dsl.Output[dsl.Dataset]` and `dsl.Input[dsl.Dataset]` create an explicit producer-to-consumer dependency. | +| Artifact persistence | Supported | The launcher uploads artifact files and metadata to the configured S3-compatible store and downloads them for consumers. | + +The verification pipeline linked below produced one `Dataset`, read it from +three `ParallelFor` tasks, and then submitted the identical pipeline again. All +three consumers succeeded, and four executor tasks in the second run were +reported as `SKIPPED` because their cached results were reused. The dataset +object remained present in S3 after the workflow pods completed. + +:::note +The managed AutoGluon pipelines intentionally disable caching for their main +tasks. Their source object key can refer to mutable data, and each run must +publish fresh progress and model artifacts. Caching remains available to your +own pipelines. +::: + +## Cache a task safely + +Caching is enabled for a run by default. It can also be set when submitting a +compiled package: + +```python +client.create_run_from_pipeline_package( + pipeline_file="pipeline.yaml", + arguments={"dataset_version": "2026-07-23"}, + enable_caching=True, +) +``` + +Disable caching for a task with external side effects or mutable inputs: + +```python +task = publish_report(...) +task.set_caching_options(False) +``` + +KFP computes a cache key from the component and its declared inputs. It cannot +detect that the contents behind an unchanged S3 object key, URL, database +query, or image tag changed. Pass an immutable object version, digest, or +explicit version parameter when fresh external content must invalidate the +cache. + +## Run tasks with `ParallelFor` + +Use `dsl.ParallelFor` when the same component can process independent items: + +```python +from kfp import dsl + + +@dsl.pipeline(name="parallel-evaluation") +def evaluation_pipeline(): + dataset = produce_dataset() + with dsl.ParallelFor( + items=["accuracy", "precision", "recall"], + parallelism=3, + ) as metric: + evaluate(dataset=dataset.outputs["dataset"], metric=metric) +``` + +The controller creates one `evaluate` task for each item. Set `parallelism` +below the item count when namespace quota, node capacity, or an external API +cannot sustain every iteration at once. + +## Persist and pass a typed artifact + +Write files to the path supplied by the output artifact. Do not invent a local +path and expect it to survive the producer pod: + +```python +from kfp import dsl + + +@dsl.component(base_image="python:3.12-slim") +def produce_dataset(dataset: dsl.Output[dsl.Dataset]): + from pathlib import Path + + output = Path(dataset.path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text("value\n42\n", encoding="utf-8") + dataset.metadata["rows"] = 1 + + +@dsl.component(base_image="python:3.12-slim") +def consume_dataset(dataset: dsl.Input[dsl.Dataset]): + from pathlib import Path + + print(Path(dataset.path).read_text(encoding="utf-8")) +``` + +Passing `producer.outputs["dataset"]` to a consumer establishes the dependency. +The KFP launcher uploads the producer's file to the artifact store and +materializes it at `dataset.path` in the consumer pod. + +This is different from a KFP workspace PVC: + +| Storage | Best for | Lifetime | +|---|---|---| +| Typed KFP artifact | Declared component inputs and outputs, lineage, results that must remain after the run | Persisted in the configured object store according to its retention policy | +| Workspace PVC | Large intermediate files shared by tasks in one run | Created for the run; lifecycle follows KFP workspace handling | +| Container filesystem | Temporary scratch data inside one task | Lost when the pod is removed | + +## Run the verification pipeline + +Download +[`verify-kfp-features.py`](https://github.com/alauda/aml-docs/tree/master/docs/en/training_guides/assets/pipelines/verify-kfp-features.py) +and run it from a Workbench or another environment that can reach the KFP API: + +```bash +python -m pip install 'kfp==2.16.1' + +export KFP_ENDPOINT='http://:8888' +export KFP_NAMESPACE='my-pipeline-namespace' +python verify-kfp-features.py +``` + +The script compiles one producer and three parallel consumers, submits it +twice with the same parameter, and fails unless all of the following are true: + +- the first run succeeds; +- exactly three `verify-dataset` tasks succeed; +- every consumer reads the producer's Dataset content; and +- the second run reports cached tasks as `SKIPPED`. + +The final output contains one `PASS` line for each mechanism. An administrator +can independently confirm persistence by listing the run prefix in the KFP +artifact bucket after the pods finish. + +## Platform compatibility and database backends + +The following results apply to the Alauda builds aligned with KFP 2.16.1: + +| Installation | x86 | arm64 | MySQL | PostgreSQL | +|---|---|---|---|---| +| DSPO 2.15.1 | Operator and DSPA verified; smoke and mechanism tests pass | Operator manager starts, but DSPA operands cannot start | Supported | Not supported | +| `kfp-operator` 26.3.0 | Operator and managed KFP installation supported | Not supported in the current image set | Supported | Not supported | + +On arm64, the current `kfp-metadata-envoy`, persistence agent, scheduled +workflow controller, API server, and pipelines-components images do not contain +an arm64 manifest. The DSPO manager being `Ready` therefore does not establish +that a pipeline service is usable. + +Both operators currently expose only MySQL connection settings. DSPO performs +its external database health check with the MySQL driver and renders +`DB_DRIVER_NAME=mysql`. The `kfp-operator` chart renders `dbType=mysql`, +`DBDriverName=mysql`, and `DBCONFIG_MYSQLCONFIG_*`. Pointing either operator at +a PostgreSQL endpoint does not switch its driver and does not produce a usable +KFP installation. Use MySQL until a PostgreSQL driver selection and the +upstream PostgreSQL deployment patches are wired into the operator interface. diff --git a/docs/en/training_guides/reusable-pipeline-components.mdx b/docs/en/training_guides/reusable-pipeline-components.mdx new file mode 100644 index 0000000..77a4148 --- /dev/null +++ b/docs/en/training_guides/reusable-pipeline-components.mdx @@ -0,0 +1,209 @@ +--- +weight: 58 +--- + +# Use Reusable Kubeflow Pipeline Components + +Alauda AI packages reusable components from the mirrored +[`pipelines-components`](https://gitlab-ce.alauda.cn/aml/pipelines-components) +project with Kubeflow Pipelines 2.16.1. A Kubeflow Pipelines installation +preloads two beta AutoGluon pipelines: + +| Pipeline | Use it for | Input data | +|---|---|---| +| `autogluon-tabular-training-pipeline` | Regression, binary classification, and multiclass classification | CSV with a target column | +| `autogluon-timeseries-training-pipeline` | Forecasting one or more time series | CSV or Parquet with ID, timestamp, and numeric target columns | + +The managed pipelines are the shortest path to a complete training run. You can +also clone the mirror and import individual components when you need a custom +pipeline DAG. + +:::warning +The current pipeline runtime and several Kubeflow Pipelines service images are +available for `linux/amd64` only. The DSPO manager can install on an arm64 +cluster, but a `DataSciencePipelinesApplication` cannot become ready there. +Run these pipelines on x86 worker nodes. +::: + +## Prerequisites \{#prerequisites\} + +| Requirement | Details | +|---|---| +| Kubeflow Pipelines 2.16.1 | Installed by either DSPO or `kfp-operator`. Do not install both operators on the same cluster. | +| S3-compatible input storage | The data loader reads its input object directly from S3. | +| S3 credentials Secret | Create it in the namespace where the pipeline run executes. | +| Workspace storage | The managed pipelines request a 12 GiB `ReadWriteOnce` workspace PVC. The installation must configure a usable default storage class. | +| Compute | The data loader requests 2 CPU and 8 GiB. Training with `preset=speed` requests 4 CPU and 16 GiB; `preset=balanced` requests 8 CPU and 32 GiB. | + +The input bucket can differ from the object store configured as the KFP artifact +store. The Secret below authenticates the data loader to the input bucket; KFP +supplies artifact-store credentials to its launcher separately. + +## Create the input-storage Secret + +Create the Secret in the pipeline run namespace: + +```bash +NS=my-pipeline-namespace + +kubectl -n "$NS" create secret generic pipeline-input-s3 \ + --from-literal=AWS_ACCESS_KEY_ID='' \ + --from-literal=AWS_SECRET_ACCESS_KEY='' \ + --from-literal=AWS_S3_ENDPOINT='https://s3.example.com' \ + --from-literal=AWS_DEFAULT_REGION='us-east-1' +``` + +Keep the URI scheme in `AWS_S3_ENDPOINT`. The component passes this value to +the S3 client as its endpoint URL. + +Upload the training file to the bucket before starting the run. For example, +the following tabular input has two features and a regression label: + +```csv +feature_a,feature_b,price +1.0,2.0,10.5 +2.0,3.0,20.5 +3.0,4.0,30.5 +``` + +Tabular data must contain at least 100 valid rows after duplicate rows, invalid +labels, and other unusable values are removed. + +## Run a managed tabular pipeline + +1. Open **Kubeflow Pipelines** and select **Pipelines**. +2. Search for `autogluon-tabular-training-pipeline` and select its latest version. +3. Select **Create run**, choose the namespace containing the Secret, and set the inputs: + +| Input | Example | Description | +|---|---|---| +| `train_data_secret_name` | `pipeline-input-s3` | S3 credentials Secret | +| `train_data_bucket_name` | `training-data` | Input bucket | +| `train_data_file_key` | `tabular/housing.csv` | CSV object key | +| `label_column` | `price` | Target column | +| `task_type` | `regression` | `regression`, `binary`, or `multiclass` | +| `top_n` | `3` | Number of models to retain, from 1 through 10 | +| `positive_class` | empty | Optional positive label for binary classification | +| `eval_metric` | empty | Empty selects `r2` for regression and `accuracy` for classification | +| `preset` | `speed` | `speed` or the larger `balanced` resource tier | + +4. Start the run and follow the data-loader and model-training tasks in the run graph. + +The pipeline samples and splits the source data, trains and ranks AutoGluon +models, and refits the best `top_n` models. Training splits are shared through +the run's workspace PVC. The test split, leaderboard, model predictors, +metrics, and generated inference notebooks are KFP artifacts persisted in the +configured artifact store. + +## Run a managed time-series pipeline + +Select `autogluon-timeseries-training-pipeline` and provide the common storage +inputs described above. Its time-series-specific inputs are: + +| Input | Example | Description | +|---|---|---| +| `target` | `sales` | Numeric value to forecast | +| `id_column` | `product_id` | Identifies each independent series | +| `timestamp_column` | `date` | Parseable observation timestamp | +| `known_covariates_names` | `["is_holiday", "promo"]` | Optional columns known for the forecast horizon | +| `prediction_length` | `14` | Number of time steps to forecast | +| `top_n` | `3` | Number of models to retain | +| `eval_metric` | `mean_absolute_scaled_error` | AutoGluon time-series ranking metric | +| `preset` | `speed` | `speed` or `balanced` | + +The loader deduplicates ID/timestamp pairs and performs a temporal split for +each series. It writes the working train splits to the workspace PVC and the +test split and trained models to the artifact store. + +## Compose a custom pipeline from the mirror + +The `kfp-components` package is not published to PyPI. Clone the Alauda mirror +and install it in a Python 3.11 or later virtual environment. Pin the SDK to the +version shipped by Alauda AI: + +```bash +git clone https://gitlab-ce.alauda.cn/aml/pipelines-components.git +cd pipelines-components + +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install 'kfp==2.16.1' 'kfp-kubernetes==2.16.1' +python -m pip install --no-deps -e . +``` + +This example imports the tabular data loader and uses it in a smaller custom +pipeline. Replace the image with the AutoML runtime mirrored into your cluster: + +```python +from kfp import compiler, dsl, kubernetes +from kfp_components.components.data_processing.automl.tabular_data_loader import ( + automl_data_loader, +) + +AUTOML_IMAGE = "/mlops/kubeflow/odh-automl:odh-stable-20260720" + + +@dsl.pipeline( + name="custom-tabular-loader", + pipeline_config=dsl.PipelineConfig( + workspace=dsl.WorkspaceConfig( + size="12Gi", + kubernetes=dsl.KubernetesWorkspaceConfig( + pvcSpecPatch={"accessModes": ["ReadWriteOnce"]} + ), + ) + ), +) +def custom_tabular_loader( + secret_name: str, + bucket: str, + object_key: str, + label: str, + task_type: str = "regression", +): + loader = automl_data_loader( + bucket_name=bucket, + file_key=object_key, + workspace_path=dsl.WORKSPACE_PATH_PLACEHOLDER, + label_column=label, + task_type=task_type, + ) + loader.set_container_image(AUTOML_IMAGE) + kubernetes.use_secret_as_env( + loader, + secret_name=secret_name, + secret_key_to_env={ + "AWS_ACCESS_KEY_ID": "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY": "AWS_SECRET_ACCESS_KEY", + "AWS_S3_ENDPOINT": "AWS_S3_ENDPOINT", + "AWS_DEFAULT_REGION": "AWS_DEFAULT_REGION", + }, + ) + + +compiler.Compiler().compile( + pipeline_func=custom_tabular_loader, + package_path="custom-tabular-loader.yaml", +) +``` + +Upload `custom-tabular-loader.yaml` from the Kubeflow Pipelines UI, or submit +it with the KFP SDK. Review the component's `README.md`, `metadata.yaml`, and +tests in the mirror before using another asset; the repository contains alpha +and beta assets with different external-service requirements. + +## Troubleshooting + +- **Secret not found:** Secrets are namespace-scoped. Create the input Secret + in the same namespace as the run. +- **Workspace PVC remains Pending:** Configure a default storage class that + supports `ReadWriteOnce`, or ask the platform administrator to configure the + KFP workspace storage class. +- **Run remains Pending:** Compare available cluster resources with the CPU and + memory requests in [Prerequisites](#prerequisites). Start with `preset=speed`. +- **Input object cannot be read:** Check the endpoint scheme, bucket and object + key, credentials, region, network policy, and the S3 server certificate. +- **Artifacts are missing after pods finish:** The pipeline input Secret does + not configure KFP persistence. Check the KFP installation's artifact-store + endpoint and credentials with the platform administrator. From b3b0e1e65d0a25845eba0d1305b866c319c34f95 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Thu, 23 Jul 2026 15:22:29 +0800 Subject: [PATCH 2/3] docs: describe arm64 pipeline operator deployment --- .../kfp-execution-and-storage.mdx | 59 +++++++++++++++++++ .../reusable-pipeline-components.mdx | 6 +- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/docs/en/training_guides/kfp-execution-and-storage.mdx b/docs/en/training_guides/kfp-execution-and-storage.mdx index 1baa77a..bb96176 100644 --- a/docs/en/training_guides/kfp-execution-and-storage.mdx +++ b/docs/en/training_guides/kfp-execution-and-storage.mdx @@ -144,6 +144,61 @@ The final output contains one `PASS` line for each mechanism. An administrator can independently confirm persistence by listing the run prefix in the KFP artifact bucket after the pods finish. +## Deploy on an arm64 cluster \{#arm64-deployment\} + +An arm64 deployment must have an arm64 manifest for every image in the selected +release. Disabling the optional KFP metadata writer removes one amd64-only image; +it does not make other amd64-only images runnable. Check the release image list +before creating the operator instance. + +### `kfp-operator` + +When the selected KFP release supports arm64, install `kfp-operator` normally +from OperatorHub and set the optional metadata writer to `false` in the +`KubeflowPipelines` custom resource: + +```yaml +apiVersion: operator.alauda.io/v1 +kind: KubeflowPipelines +metadata: + name: kubeflowpipelines +spec: + global: + images: + kfpMetadataWriter: + enabled: false +``` + +Keep the remaining `spec.global.images` values from the release sample. The +`support_arm` image field only controls packaging/image relocation; it does not +disable the Deployment. Verify that the writer is absent after reconciliation: + +```bash +kubectl -n kubeflow get deployment metadata-writer +# Error from server (NotFound) is expected +``` + +### Data Science Pipelines Operator (DSPO) + +DSPO's supported V2 reconciler does not deploy `kfp-metadata-writer` at all. +Create the DSPA with the normal V2 settings and do not add an MLMD writer field: + +```yaml +apiVersion: datasciencepipelinesapplications.opendatahub.io/v1 +kind: DataSciencePipelinesApplication +metadata: + name: sample + namespace: data-science-project +spec: + dspVersion: v2 + mlmd: + deploy: true +``` + +Verify the namespace contains MLMD gRPC/Envoy resources but no +`metadata-writer` Deployment. DSPO and `kfp-operator` are mutually exclusive; +choose one installation model per cluster. + ## Platform compatibility and database backends The following results apply to the Alauda builds aligned with KFP 2.16.1: @@ -158,6 +213,10 @@ workflow controller, API server, and pipelines-components images do not contain an arm64 manifest. The DSPO manager being `Ready` therefore does not establish that a pipeline service is usable. +The arm64 instructions above only address the optional metadata writer. They do +not override this image compatibility requirement; use a release whose complete +runtime image set has arm64 manifests before attempting a DSPA or KFP deployment. + Both operators currently expose only MySQL connection settings. DSPO performs its external database health check with the MySQL driver and renders `DB_DRIVER_NAME=mysql`. The `kfp-operator` chart renders `dbType=mysql`, diff --git a/docs/en/training_guides/reusable-pipeline-components.mdx b/docs/en/training_guides/reusable-pipeline-components.mdx index 77a4148..3e5fa74 100644 --- a/docs/en/training_guides/reusable-pipeline-components.mdx +++ b/docs/en/training_guides/reusable-pipeline-components.mdx @@ -22,7 +22,11 @@ pipeline DAG. The current pipeline runtime and several Kubeflow Pipelines service images are available for `linux/amd64` only. The DSPO manager can install on an arm64 cluster, but a `DataSciencePipelinesApplication` cannot become ready there. -Run these pipelines on x86 worker nodes. +Run these pipelines on x86 worker nodes. If a later release supplies arm64 +manifests for the complete runtime image set, follow the [arm64 deployment +instructions](./kfp-execution-and-storage.mdx#arm64-deployment); DSPO does not +deploy the optional metadata writer, while `kfp-operator` requires +`kfpMetadataWriter.enabled: false`. ::: ## Prerequisites \{#prerequisites\} From 35524841f941e036ba371194a9c42f56a4a49a02 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Fri, 24 Jul 2026 15:51:52 +0800 Subject: [PATCH 3/3] docs: update KFP arm64 release guidance --- .../kfp-execution-and-storage.mdx | 60 +++++++++---------- .../reusable-pipeline-components.mdx | 27 ++++----- 2 files changed, 42 insertions(+), 45 deletions(-) diff --git a/docs/en/training_guides/kfp-execution-and-storage.mdx b/docs/en/training_guides/kfp-execution-and-storage.mdx index bb96176..8c7756d 100644 --- a/docs/en/training_guides/kfp-execution-and-storage.mdx +++ b/docs/en/training_guides/kfp-execution-and-storage.mdx @@ -4,26 +4,28 @@ weight: 59 # Kubeflow Pipelines Execution and Storage Behavior -Kubeflow Pipelines 2.16.1 installed by the Data Science Pipelines Operator -(DSPO) supports task caching, `dsl.ParallelFor`, typed KFP artifacts, and -S3-compatible artifact persistence on x86 clusters. These mechanisms are KFP -features; DSPO configures the API server, workflow controller, cache server, -and artifact store that implement them. +Kubeflow Pipelines 2.16.1 installed by Alauda `kfp-operator` v26.3.1 or the +Data Science Pipelines Operator (DSPO) v2.15.2 supports task caching, +`dsl.ParallelFor`, typed KFP artifacts, and S3-compatible artifact persistence +on amd64 and arm64 clusters. These mechanisms are KFP features; the operator +configures the API server, workflow controller, cache server, and artifact +store that implement them. ## Feature summary -| Mechanism | DSPO on x86 | Behavior | +| Mechanism | KFP 2.16.1 | Behavior | |---|---|---| | Task caching | Supported | An identical task execution can be reused and is reported as `SKIPPED` in the repeated run. | | `dsl.ParallelFor` | Supported | Each loop item becomes a separate task; `parallelism` limits concurrent iterations. | | Typed artifacts | Supported | `dsl.Output[dsl.Dataset]` and `dsl.Input[dsl.Dataset]` create an explicit producer-to-consumer dependency. | | Artifact persistence | Supported | The launcher uploads artifact files and metadata to the configured S3-compatible store and downloads them for consumers. | -The verification pipeline linked below produced one `Dataset`, read it from -three `ParallelFor` tasks, and then submitted the identical pipeline again. All -three consumers succeeded, and four executor tasks in the second run were -reported as `SKIPPED` because their cached results were reused. The dataset -object remained present in S3 after the workflow pods completed. +The verification pipeline linked below was run with DSPO on an x86 cluster. It +produced one `Dataset`, read it from three `ParallelFor` tasks, and then +submitted the identical pipeline again. All three consumers succeeded, and +four executor tasks in the second run were reported as `SKIPPED` because their +cached results were reused. The dataset object remained present in S3 after +the workflow pods completed. :::note The managed AutoGluon pipelines intentionally disable caching for their main @@ -146,16 +148,16 @@ artifact bucket after the pods finish. ## Deploy on an arm64 cluster \{#arm64-deployment\} -An arm64 deployment must have an arm64 manifest for every image in the selected -release. Disabling the optional KFP metadata writer removes one amd64-only image; -it does not make other amd64-only images runnable. Check the release image list -before creating the operator instance. +The Alauda KFP 2.16.1 releases package the pipeline service and runtime images +for both `linux/amd64` and `linux/arm64`. The optional +`kfp-metadata-writer` image remains amd64-only. DSPO does not deploy this +component; Alauda `kfp-operator` must disable it on arm64. ### `kfp-operator` -When the selected KFP release supports arm64, install `kfp-operator` normally -from OperatorHub and set the optional metadata writer to `false` in the -`KubeflowPipelines` custom resource: +With Alauda `kfp-operator` v26.3.1, install the operator normally from +OperatorHub and set the optional metadata writer to `false` in the +`KubeflowPipelines` custom resource before reconciliation: ```yaml apiVersion: operator.alauda.io/v1 @@ -180,8 +182,9 @@ kubectl -n kubeflow get deployment metadata-writer ### Data Science Pipelines Operator (DSPO) -DSPO's supported V2 reconciler does not deploy `kfp-metadata-writer` at all. -Create the DSPA with the normal V2 settings and do not add an MLMD writer field: +The V2 reconciler in DSPO v2.15.2 does not deploy `kfp-metadata-writer`. No +arm64 override is required. Create the DSPA with the normal V2 settings and do +not add an MLMD writer field: ```yaml apiVersion: datasciencepipelinesapplications.opendatahub.io/v1 @@ -203,19 +206,14 @@ choose one installation model per cluster. The following results apply to the Alauda builds aligned with KFP 2.16.1: -| Installation | x86 | arm64 | MySQL | PostgreSQL | +| Installation | amd64 | arm64 | MySQL | PostgreSQL | |---|---|---|---|---| -| DSPO 2.15.1 | Operator and DSPA verified; smoke and mechanism tests pass | Operator manager starts, but DSPA operands cannot start | Supported | Not supported | -| `kfp-operator` 26.3.0 | Operator and managed KFP installation supported | Not supported in the current image set | Supported | Not supported | +| DSPO v2.15.2 | Supported; smoke and mechanism tests pass | Supported; DSPA readiness and KFP v2 SDK smoke pass | Supported | Not supported | +| Alauda `kfp-operator` v26.3.1 | Supported | Supported when `kfpMetadataWriter.enabled` is `false`; KFP API and Argo workflow execution verified | Supported | Not supported | -On arm64, the current `kfp-metadata-envoy`, persistence agent, scheduled -workflow controller, API server, and pipelines-components images do not contain -an arm64 manifest. The DSPO manager being `Ready` therefore does not establish -that a pipeline service is usable. - -The arm64 instructions above only address the optional metadata writer. They do -not override this image compatibility requirement; use a release whose complete -runtime image set has arm64 manifests before attempting a DSPA or KFP deployment. +All required runtime images in these releases have amd64 and arm64 manifests, +except `kfp-metadata-writer`. Disabling that component is required only for an +arm64 Alauda `kfp-operator` installation; it is not part of a DSPO deployment. Both operators currently expose only MySQL connection settings. DSPO performs its external database health check with the MySQL driver and renders diff --git a/docs/en/training_guides/reusable-pipeline-components.mdx b/docs/en/training_guides/reusable-pipeline-components.mdx index 3e5fa74..2526a2b 100644 --- a/docs/en/training_guides/reusable-pipeline-components.mdx +++ b/docs/en/training_guides/reusable-pipeline-components.mdx @@ -4,10 +4,11 @@ weight: 58 # Use Reusable Kubeflow Pipeline Components -Alauda AI packages reusable components from the mirrored +The next Alauda AI release packages reusable components from the mirrored [`pipelines-components`](https://gitlab-ce.alauda.cn/aml/pipelines-components) -project with Kubeflow Pipelines 2.16.1. A Kubeflow Pipelines installation -preloads two beta AutoGluon pipelines: +project for Kubeflow Pipelines 2.16.1. This KFP version will be delivered by +Alauda `kfp-operator` v26.3.1 and DSPO v2.15.2. Either installation preloads +two beta AutoGluon pipelines: | Pipeline | Use it for | Input data | |---|---|---| @@ -18,22 +19,20 @@ The managed pipelines are the shortest path to a complete training run. You can also clone the mirror and import individual components when you need a custom pipeline DAG. -:::warning -The current pipeline runtime and several Kubeflow Pipelines service images are -available for `linux/amd64` only. The DSPO manager can install on an arm64 -cluster, but a `DataSciencePipelinesApplication` cannot become ready there. -Run these pipelines on x86 worker nodes. If a later release supplies arm64 -manifests for the complete runtime image set, follow the [arm64 deployment -instructions](./kfp-execution-and-storage.mdx#arm64-deployment); DSPO does not -deploy the optional metadata writer, while `kfp-operator` requires -`kfpMetadataWriter.enabled: false`. +:::note +The KFP 2.16.1 service and reusable-component images support `linux/amd64` and +`linux/arm64`. DSPO needs no additional setting on arm64 because it does not +deploy the optional, amd64-only metadata writer. For Alauda `kfp-operator`, set +`kfpMetadataWriter.enabled: false`. See the [arm64 deployment +instructions](./kfp-execution-and-storage.mdx#arm64-deployment). ::: ## Prerequisites \{#prerequisites\} | Requirement | Details | |---|---| -| Kubeflow Pipelines 2.16.1 | Installed by either DSPO or `kfp-operator`. Do not install both operators on the same cluster. | +| Kubeflow Pipelines 2.16.1 | Will be included in the next Alauda AI release with either Alauda `kfp-operator` v26.3.1 or DSPO v2.15.2. Install only one pipeline operator on a cluster. | +| Supported architecture | `linux/amd64` or `linux/arm64`. On arm64, follow the operator-specific configuration linked above. | | S3-compatible input storage | The data loader reads its input object directly from S3. | | S3 credentials Secret | Create it in the namespace where the pipeline run executes. | | Workspace storage | The managed pipelines request a 12 GiB `ReadWriteOnce` workspace PVC. The installation must configure a usable default storage class. | @@ -145,7 +144,7 @@ from kfp_components.components.data_processing.automl.tabular_data_loader import automl_data_loader, ) -AUTOML_IMAGE = "/mlops/kubeflow/odh-automl:odh-stable-20260720" +AUTOML_IMAGE = "/mlops/kubeflow/odh-automl:v1.11.0.post.1" @dsl.pipeline(