From cb1f31f83635486e3fc2a59775c61a92caaee1a5 Mon Sep 17 00:00:00 2001 From: Wenjia218 Date: Thu, 13 Aug 2026 15:59:32 +0200 Subject: [PATCH 1/5] add kendall tau metric --- src/metrics/kendall_tau/config.vsh.yaml | 78 +++++++++++++++++++++++++ src/metrics/kendall_tau/script.py | 68 +++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 src/metrics/kendall_tau/config.vsh.yaml create mode 100644 src/metrics/kendall_tau/script.py diff --git a/src/metrics/kendall_tau/config.vsh.yaml b/src/metrics/kendall_tau/config.vsh.yaml new file mode 100644 index 0000000..7ba3e80 --- /dev/null +++ b/src/metrics/kendall_tau/config.vsh.yaml @@ -0,0 +1,78 @@ +__merge__: ../../api/comp_metric.yaml + +name: kendall_tau + +info: + metrics: + + - name: kendall_tau + label: Kendall Tau + summary: "Computes the Kendall's τ coefficient between inferred and ground-truth pseudotime." + description: | + Calculates the Kendall's τ coefficient between predicted and ground-truth pseudotime. + Kendall's τ (tau) is a non-parametric statistic that measures the ordinal association, or rank correlation, between two variables based on how similarly they order a set of observations. + references: + doi: + - 10.1038/s41592-020-0772-5 + bibtex: | + @article{Virtanen_2020, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and Haberland, Matt and Reddy, Tyler and Cournapeau, David and Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and Bright, Jonathan and {van der Walt}, St{\'e}fan J. and Brett, Matthew and Wilson, Joshua and Jarrod Millman, K. and Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and Kern, Robert and Larson, Eric and Carey, C. J. and Polat, {\dot{I}}lhan and Feng, Yu and Moore, Eric W. and VanderPlas, Jake and Laxalde, Denis and Perktold, Josef and Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and Harris, Charles R. and Archibald, Anne M. and Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, + title = {Author Correction: SciPy 1.0: fundamental algorithms for scientific computing in Python}, + journal = {Nature Methods}, + volume = {17}, + number = {3}, + pages = {352}, + year = {2020}, + doi = {10.1038/s41592-020-0772-5} + } + links: + # URL to the documentation for this metric (required). + documentation: https://url.to/the/documentation + # URL to the code repository for this metric (required). + repository: https://github.com/organisation/repository + # The minimum possible value for this metric (required) + min: -1 + # The maximum possible value for this metric (required) + max: 1 + # Whether a higher value represents a 'better' solution (required) + maximize: true + +# Component-specific parameters (optional) +# arguments: +# - name: "--n_neighbors" +# type: "integer" +# default: 5 +# description: Number of neighbors to use. + +# Resources required to run the component +resources: + - type: python_script + path: script.py + + +engines: + - type: docker + image: python:3.11-slim + setup: + - type: apt + packages: + - procps # required by Nextflow + - git # pip needs it to install openproblems core from git+https + - type: python + packages: + - anndata~=0.10.0 + - scanpy~=1.10.0 + - scipy + - pandas + - pyyaml + - requests + - jsonschema + github: + - "openproblems-bio/core#subdirectory=packages/python/openproblems" + + +runners: + - type: executable + - type: nextflow + directives: + label: [midtime,midmem,midcpu] diff --git a/src/metrics/kendall_tau/script.py b/src/metrics/kendall_tau/script.py new file mode 100644 index 0000000..59c2332 --- /dev/null +++ b/src/metrics/kendall_tau/script.py @@ -0,0 +1,68 @@ +import anndata as ad +import numpy as np +import pandas as pd +from scipy import stats + +## VIASH START +# Note: this section is auto-generated by viash at runtime. To edit it, make changes +# in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. +par = { + 'input_solution': 'resources_test/task_template/cxg_mouse_pancreas_atlas/solution.h5ad', + 'input_prediction': 'resources_test/task_template/cxg_mouse_pancreas_atlas/prediction.h5ad', + 'output': 'output.h5ad' +} + +meta = { + 'name': 'kendall_tau' +} +## VIASH END + + +def compute_kendall_tau(true_values, inferred_values): + """Calculates Kendall tau rank correlation on finite overlapping values.""" + mask = np.isfinite(true_values) & np.isfinite(inferred_values) + + if mask.sum() < 2: + return 0.0 + kendall_tau, _ = stats.kendalltau(true_values[mask], inferred_values[mask]) + + if np.isnan(kendall_tau): + return 0.0 + + return float(kendall_tau) + +print('Reading input files', flush=True) +input_solution = ad.read_h5ad(par['input_solution']) +input_prediction = ad.read_h5ad(par['input_prediction']) + +assert (input_prediction.obs_names == input_solution.obs_names).all(), "obs_names not the same in prediction and solution inputs" + +# ground truth and predicted pseudotime +TRUE_COL = "pseudotime_true" +INFERRED_COL = "pseudotime_inferred" +true_vals = pd.to_numeric(input_solution.obs[TRUE_COL], errors='coerce').values +inferred_vals = pd.to_numeric(input_prediction.obs[INFERRED_COL], errors='coerce').values + + +score = compute_kendall_tau(true_vals, inferred_vals) + + +# metric_ids and metric_values can have length > 1 +# but should be of equal length +uns_metric_ids = [ 'kendall_tau' ] +uns_metric_values = [ score ] + +print("Write output AnnData to file", flush=True) +output = ad.AnnData( + obs=pd.DataFrame(index=pd.Index(np.array([], dtype=str))), + var=pd.DataFrame(index=pd.Index(np.array([], dtype=str))), + uns={ + 'dataset_id': input_solution.uns.get('dataset_id', 'unknown'), + 'normalization_id': input_solution.uns.get('normalization_id', 'unknown'), + 'method_id': input_prediction.uns.get('method_id', 'unknown'), + 'metric_ids': uns_metric_ids, + 'metric_values': uns_metric_values, + } +) + +output.write_h5ad(par['output'], compression='gzip') From e2d59dba9251f3f1ceef3ab8e5f02d1316bf0736 Mon Sep 17 00:00:00 2001 From: Wenjia218 Date: Thu, 13 Aug 2026 16:06:06 +0200 Subject: [PATCH 2/5] remove comments --- src/metrics/kendall_tau/config.vsh.yaml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/metrics/kendall_tau/config.vsh.yaml b/src/metrics/kendall_tau/config.vsh.yaml index 7ba3e80..1bb4cbd 100644 --- a/src/metrics/kendall_tau/config.vsh.yaml +++ b/src/metrics/kendall_tau/config.vsh.yaml @@ -26,25 +26,12 @@ info: doi = {10.1038/s41592-020-0772-5} } links: - # URL to the documentation for this metric (required). documentation: https://url.to/the/documentation - # URL to the code repository for this metric (required). repository: https://github.com/organisation/repository - # The minimum possible value for this metric (required) min: -1 - # The maximum possible value for this metric (required) max: 1 - # Whether a higher value represents a 'better' solution (required) maximize: true -# Component-specific parameters (optional) -# arguments: -# - name: "--n_neighbors" -# type: "integer" -# default: 5 -# description: Number of neighbors to use. - -# Resources required to run the component resources: - type: python_script path: script.py From 89bd1db3b7afcb37be786cab2c6872b3921e9f3c Mon Sep 17 00:00:00 2001 From: Wenjia218 Date: Fri, 14 Aug 2026 11:27:02 +0200 Subject: [PATCH 3/5] add package version --- src/metrics/kendall_tau/config.vsh.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/metrics/kendall_tau/config.vsh.yaml b/src/metrics/kendall_tau/config.vsh.yaml index 1bb4cbd..233b071 100644 --- a/src/metrics/kendall_tau/config.vsh.yaml +++ b/src/metrics/kendall_tau/config.vsh.yaml @@ -36,7 +36,6 @@ resources: - type: python_script path: script.py - engines: - type: docker image: python:3.11-slim @@ -47,13 +46,14 @@ engines: - git # pip needs it to install openproblems core from git+https - type: python packages: - - anndata~=0.10.0 - - scanpy~=1.10.0 - - scipy - - pandas - - pyyaml - - requests - - jsonschema + - anndata~=0.10.9 + - scanpy~=1.10.4 + - numpy~=2.4.6 + - pandas~=3.0.5 + - scipy~=1.17.1 + - pyyaml~=6.0.3 + - requests~=2.34.2 + - jsonschema~=4.26.0 github: - "openproblems-bio/core#subdirectory=packages/python/openproblems" From fce4073f6774091470293d03fc9c0c3280885a10 Mon Sep 17 00:00:00 2001 From: Wenjia218 Date: Fri, 14 Aug 2026 12:24:43 +0200 Subject: [PATCH 4/5] add geodestic distance metric --- src/metrics/geodesic_distance/config.vsh.yaml | 64 +++++++++++++++++++ src/metrics/geodesic_distance/script.py | 30 +++++++++ src/metrics/kendall_tau/config.vsh.yaml | 4 +- 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 src/metrics/geodesic_distance/config.vsh.yaml create mode 100644 src/metrics/geodesic_distance/script.py diff --git a/src/metrics/geodesic_distance/config.vsh.yaml b/src/metrics/geodesic_distance/config.vsh.yaml new file mode 100644 index 0000000..57a2ab9 --- /dev/null +++ b/src/metrics/geodesic_distance/config.vsh.yaml @@ -0,0 +1,64 @@ +__merge__: ../../api/comp_metric.yaml + +name: geodesic_distance + +info: + metrics: + + - name: geodesic_distance + label: Geodesic Distance Correlation + summary: "Computes the Spearman correlation between pairwise geodesic distances of inferred and ground-truth pseudotime." + description: | + Calculates the Spearman rank correlation between all pairwise distances of predicted pseudotime and all pairwise distances of ground-truth pseudotime, computed over every unique pair of cells. + This measures whether the relative spacing between cells along the trajectory is preserved and captures distortions where a method compresses or stretches segments of the trajectory even if the overall cell ordering is correct. + references: + doi: + - 10.1038/s41592-020-0772-5 + bibtex: | + @article{Virtanen_2020, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and Haberland, Matt and Reddy, Tyler and Cournapeau, David and Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and Bright, Jonathan and {van der Walt}, St{\'e}fan J. and Brett, Matthew and Wilson, Joshua and Jarrod Millman, K. and Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and Kern, Robert and Larson, Eric and Carey, C. J. and Polat, {\dot{I}}lhan and Feng, Yu and Moore, Eric W. and VanderPlas, Jake and Laxalde, Denis and Perktold, Josef and Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and Harris, Charles R. and Archibald, Anne M. and Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, + title = {Author Correction: SciPy 1.0: fundamental algorithms for scientific computing in Python}, + journal = {Nature Methods}, + volume = {17}, + number = {3}, + pages = {352}, + year = {2020}, + doi = {10.1038/s41592-020-0772-5} + } + links: + documentation: https://docs.scipy.org/doc/scipy/reference/spatial.distance.html + repository: https://github.com/scipy/scipy + min: -1 + max: 1 + maximize: true + +resources: + - type: python_script + path: script.py + +engines: + - type: docker + image: python:3.11-slim + setup: + - type: apt + packages: + - procps # required by Nextflow + - git # pip needs it to install openproblems core from git+https + - type: python + packages: + - anndata~=0.10.9 + - scanpy~=1.10.4 + - numpy~=2.4.6 + - pandas~=3.0.5 + - scipy~=1.17.1 + - pyyaml~=6.0.3 + - requests~=2.34.2 + - jsonschema~=4.26.0 + github: + - "openproblems-bio/core#subdirectory=packages/python/openproblems" + +runners: + - type: executable + - type: nextflow + directives: + label: [midtime,midmem,midcpu] diff --git a/src/metrics/geodesic_distance/script.py b/src/metrics/geodesic_distance/script.py new file mode 100644 index 0000000..cbf33d6 --- /dev/null +++ b/src/metrics/geodesic_distance/script.py @@ -0,0 +1,30 @@ +import anndata as ad + +## VIASH START +# Note: this section is auto-generated by viash at runtime. To edit it, make changes +# in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. +par = { + 'input_solution': 'resources_test/.../solution.h5ad', + 'input_prediction': 'resources_test/.../prediction.h5ad', + 'output': 'output.h5ad' +} +meta = { + 'name': 'geodesic_distance' +} +## VIASH END + +print('Reading input files', flush=True) +input_solution = ad.read_h5ad(par['input_solution']) +input_prediction = ad.read_h5ad(par['input_prediction']) + +print('Compute metrics', flush=True) +# metric_ids and metric_values can have length > 1 +# but should be of equal length +uns_metric_ids = [ 'geodesic_distance' ] +uns_metric_values = [ 0.5 ] + +print("Write output AnnData to file", flush=True) +output = ad.AnnData( + +) +output.write_h5ad(par['output'], compression='gzip') diff --git a/src/metrics/kendall_tau/config.vsh.yaml b/src/metrics/kendall_tau/config.vsh.yaml index 233b071..d0d60be 100644 --- a/src/metrics/kendall_tau/config.vsh.yaml +++ b/src/metrics/kendall_tau/config.vsh.yaml @@ -26,8 +26,8 @@ info: doi = {10.1038/s41592-020-0772-5} } links: - documentation: https://url.to/the/documentation - repository: https://github.com/organisation/repository + documentation: https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.kendalltau.html + repository: https://github.com/scipy/scipy min: -1 max: 1 maximize: true From 44cb0bff8c14de9df9e8edbb2d9ce75c9ece5425 Mon Sep 17 00:00:00 2001 From: Wenjia218 Date: Fri, 14 Aug 2026 12:37:48 +0200 Subject: [PATCH 5/5] add changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95e7dbc..24536e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ * Added `spaTrack` method (PR #4). * Added `Spearman's correlation` metric (PR #5). * Added `Moran's I` metric (PR #6). - +* Added `kendall_tau` and `geodesic distance` metric (PR #8) * Added `stlearn`method (PR #9). ## MAJOR CHANGES