From abcb8abbfccfb2c9247c35d88c1da86b0cefac25 Mon Sep 17 00:00:00 2001 From: ipezygj <231574583+ipezygj@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:06:21 +0300 Subject: [PATCH 1/4] Add percentile-bootstrap confidence intervals for aggregated metrics A mean metric over N cases is a point estimate: a mean DSC of 0.85 over 20 cases and over 2,000 cases support very different conclusions, and validation studies routinely compare methods whose intervals overlap entirely. The Metrics Reloaded recommendations call for reporting variability alongside aggregates. - utility/uncertainty.py: percentile_bootstrap_ci for the mean of per-case values (NaN-ignored, consistent with the package's masked aggregations; seeded by default so reported intervals reproduce) and stats_with_ci, which appends ci95_low/ci95_high rows to a describe() summary - ProcessEvaluation.get_stats_res now reports stats_all with the interval rows; columns unchanged, existing consumers unaffected - tests: interval brackets the mean, determinism under seed, narrowing with N, agreement with the analytic normal interval on Gaussian data, NaN handling, degenerate inputs --- MetricsReloaded/processes/overall_process.py | 9 +- MetricsReloaded/utility/uncertainty.py | 91 ++++++++++++++++++++ test/test_utility/test_uncertainty.py | 64 ++++++++++++++ 3 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 MetricsReloaded/utility/uncertainty.py create mode 100644 test/test_utility/test_uncertainty.py diff --git a/MetricsReloaded/processes/overall_process.py b/MetricsReloaded/processes/overall_process.py index 9e945a0..05ff479 100644 --- a/MetricsReloaded/processes/overall_process.py +++ b/MetricsReloaded/processes/overall_process.py @@ -184,6 +184,7 @@ from MetricsReloaded.processes.mixed_measures_processes import MultiLabelLocMeasures, MultiLabelPairwiseMeasures, MultiLabelLocSegPairwiseMeasure import warnings from MetricsReloaded.utility.utils import combine_df, merge_list_df +from MetricsReloaded.utility.uncertainty import stats_with_ci import pandas as pd import numpy as np @@ -682,9 +683,13 @@ def label_aggregation(self, option='average',dict_args={}): def get_stats_res(self): """ - Create summary statistics overall and per label available in self.stats_lab and self.stats_all + Create summary statistics overall and per label available in self.stats_lab and self.stats_all. + The overall statistics include percentile-bootstrap confidence + intervals (rows ci95_low / ci95_high) for each metric's mean, so that + aggregated results are reported with their uncertainty rather than as + bare point estimates. """ - df_stats_all = self.grouped_lab.describe() + df_stats_all = stats_with_ci(self.grouped_lab) self.stats_all = df_stats_all print(self.resdet, self.resseg) if len(self.resdet.index)==0 and len(self.resseg.index)==0: diff --git a/MetricsReloaded/utility/uncertainty.py b/MetricsReloaded/utility/uncertainty.py new file mode 100644 index 0000000..aa1ec51 --- /dev/null +++ b/MetricsReloaded/utility/uncertainty.py @@ -0,0 +1,91 @@ +""" +Uncertainty quantification - :mod:`MetricsReloaded.utility.uncertainty` +======================================================================= + +This module provides functions for reporting the uncertainty of aggregated +metric values. + +A mean metric over N cases is a point estimate: a mean DSC of 0.85 over 20 +cases and over 2,000 cases support very different conclusions, and validation +studies routinely compare methods whose intervals overlap entirely. The +Metrics Reloaded recommendations call for reporting variability alongside +aggregates; this module provides the standard non-parametric tool for it. + +.. currentmodule:: MetricsReloaded.utility.uncertainty + +.. autosummary:: + :nosignatures: + + percentile_bootstrap_ci + stats_with_ci + +""" + +import numpy as np +import pandas as pd + +__all__ = [ + "percentile_bootstrap_ci", + "stats_with_ci", +] + + +def percentile_bootstrap_ci(values, n_boot=2000, alpha=0.05, seed=42): + """Percentile-bootstrap confidence interval for the mean of per-case + metric values. + + NaN values are ignored, consistent with the masked aggregations used + elsewhere in this package. The resampling is seeded by default so that + reported intervals are reproducible across runs of the same evaluation. + + :param values: iterable of per-case metric values (may contain NaN) + :param n_boot: number of bootstrap resamples + :param alpha: 1 - confidence level (0.05 -> 95% interval) + :param seed: seed for the resampling generator; pass None for + non-deterministic resampling + :return: (lower, upper) bounds of the interval; (nan, nan) when fewer + than two non-NaN values are available + """ + arr = np.asarray(values, dtype=float) + arr = arr[~np.isnan(arr)] + if arr.size < 2: + return (np.nan, np.nan) + rng = np.random.default_rng(seed) + indices = rng.integers(0, arr.size, size=(n_boot, arr.size)) + boot_means = arr[indices].mean(axis=1) + return ( + float(np.quantile(boot_means, alpha / 2)), + float(np.quantile(boot_means, 1 - alpha / 2)), + ) + + +def stats_with_ci(df, n_boot=2000, alpha=0.05, seed=42): + """Summary statistics of a per-case results dataframe with + confidence-interval rows for each numeric metric column. + + Returns the ``df.describe()`` table augmented with two rows, + ``ci95_low`` and ``ci95_high`` (names follow the requested ``alpha``), + holding the percentile-bootstrap interval for each column's mean. + + :param df: dataframe of per-case metric values (columns = metrics) + :param n_boot: number of bootstrap resamples per column + :param alpha: 1 - confidence level + :param seed: seed for reproducible intervals + :return: describe() dataframe with appended CI rows + """ + described = df.describe() + level = int(round((1 - alpha) * 100)) + low_name = "ci%d_low" % level + high_name = "ci%d_high" % level + lows = {} + highs = {} + for col in described.columns: + series = pd.to_numeric(df[col], errors="coerce") + low, high = percentile_bootstrap_ci( + series.to_numpy(), n_boot=n_boot, alpha=alpha, seed=seed + ) + lows[col] = low + highs[col] = high + described.loc[low_name] = pd.Series(lows) + described.loc[high_name] = pd.Series(highs) + return described diff --git a/test/test_utility/test_uncertainty.py b/test/test_utility/test_uncertainty.py new file mode 100644 index 0000000..984dc29 --- /dev/null +++ b/test/test_utility/test_uncertainty.py @@ -0,0 +1,64 @@ +import numpy as np +import pandas as pd +from numpy.testing import assert_allclose + +from MetricsReloaded.utility.uncertainty import ( + percentile_bootstrap_ci, + stats_with_ci, +) + + +def test_interval_brackets_the_mean_and_is_deterministic(): + values = np.array([0.7, 0.8, 0.85, 0.9, 0.75, 0.82, 0.88, 0.79, 0.81, 0.86]) + low, high = percentile_bootstrap_ci(values) + assert low < values.mean() < high + # seeded resampling: identical call gives identical interval + assert (low, high) == percentile_bootstrap_ci(values) + + +def test_interval_narrows_with_more_cases(): + rng = np.random.default_rng(0) + small = rng.normal(0.8, 0.1, size=20) + large = np.concatenate([small] * 50) + low_s, high_s = percentile_bootstrap_ci(small) + low_l, high_l = percentile_bootstrap_ci(large) + assert (high_l - low_l) < (high_s - low_s) + + +def test_approximates_analytic_interval_for_gaussian_data(): + rng = np.random.default_rng(1) + values = rng.normal(0.5, 0.2, size=400) + low, high = percentile_bootstrap_ci(values, n_boot=5000) + sem = values.std(ddof=1) / np.sqrt(values.size) + assert_allclose(low, values.mean() - 1.96 * sem, atol=3 * sem / 10) + assert_allclose(high, values.mean() + 1.96 * sem, atol=3 * sem / 10) + + +def test_nan_values_are_ignored(): + values = [0.8, np.nan, 0.9, 0.85, np.nan, 0.82] + low, high = percentile_bootstrap_ci(values) + clean_low, clean_high = percentile_bootstrap_ci([0.8, 0.9, 0.85, 0.82]) + assert (low, high) == (clean_low, clean_high) + + +def test_degenerate_inputs_return_nan_interval(): + assert np.isnan(percentile_bootstrap_ci([])[0]) + assert np.isnan(percentile_bootstrap_ci([0.5])[1]) + assert np.isnan(percentile_bootstrap_ci([np.nan, np.nan])[0]) + + +def test_stats_with_ci_appends_interval_rows(): + df = pd.DataFrame( + { + "dsc": [0.7, 0.8, 0.85, 0.9, 0.75], + "nsd": [0.6, 0.65, 0.7, 0.72, 0.68], + } + ) + stats = stats_with_ci(df) + assert "ci95_low" in stats.index + assert "ci95_high" in stats.index + for col in ("dsc", "nsd"): + assert stats.loc["ci95_low", col] < df[col].mean() + assert stats.loc["ci95_high", col] > df[col].mean() + # describe() content preserved + assert_allclose(stats.loc["mean", "dsc"], df["dsc"].mean()) From 47bd50f185e70082af81b4d13cb8e5670bca5d6d Mon Sep 17 00:00:00 2001 From: ipezygj Date: Sun, 9 Aug 2026 06:59:29 +0300 Subject: [PATCH 2/4] Address review: validate CI options, exact level labels, exclude identifier columns - percentile_bootstrap_ci rejects n_boot < 1 and alpha outside (0, 1) instead of failing inside np.quantile or returning inverted bounds - interval row names preserve fractional confidence levels (alpha=0.025 -> ci97.5_low, previously mislabelled ci98) - stats_with_ci grew an exclude= option; get_stats_res passes the identifier columns so 'case'/'index' keep their describe() rows but no longer get a meaningless bootstrap interval - three new tests cover each point --- MetricsReloaded/processes/overall_process.py | 2 +- MetricsReloaded/utility/uncertainty.py | 25 +++++++++---- examples/FN_PredictionPQ.nii.gz | Bin 0 -> 118 bytes examples/FN_PredictionPQ.png | Bin 0 -> 81 bytes examples/FP_PredictionPQ.nii.gz | Bin 0 -> 133 bytes examples/FP_PredictionPQ.png | Bin 0 -> 89 bytes examples/TP_Pred_PredictionPQ.nii.gz | Bin 0 -> 161 bytes examples/TP_Pred_PredictionPQ.png | Bin 0 -> 97 bytes examples/TP_Ref_PredictionPQ.nii.gz | Bin 0 -> 162 bytes examples/TP_Ref_PredictionPQ.png | Bin 0 -> 98 bytes test/test_utility/test_uncertainty.py | 35 +++++++++++++++++++ 11 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 examples/FN_PredictionPQ.nii.gz create mode 100644 examples/FN_PredictionPQ.png create mode 100644 examples/FP_PredictionPQ.nii.gz create mode 100644 examples/FP_PredictionPQ.png create mode 100644 examples/TP_Pred_PredictionPQ.nii.gz create mode 100644 examples/TP_Pred_PredictionPQ.png create mode 100644 examples/TP_Ref_PredictionPQ.nii.gz create mode 100644 examples/TP_Ref_PredictionPQ.png diff --git a/MetricsReloaded/processes/overall_process.py b/MetricsReloaded/processes/overall_process.py index 05ff479..ac2cdb8 100644 --- a/MetricsReloaded/processes/overall_process.py +++ b/MetricsReloaded/processes/overall_process.py @@ -689,7 +689,7 @@ def get_stats_res(self): aggregated results are reported with their uncertainty rather than as bare point estimates. """ - df_stats_all = stats_with_ci(self.grouped_lab) + df_stats_all = stats_with_ci(self.grouped_lab, exclude=("index", "case")) self.stats_all = df_stats_all print(self.resdet, self.resseg) if len(self.resdet.index)==0 and len(self.resseg.index)==0: diff --git a/MetricsReloaded/utility/uncertainty.py b/MetricsReloaded/utility/uncertainty.py index aa1ec51..90a019c 100644 --- a/MetricsReloaded/utility/uncertainty.py +++ b/MetricsReloaded/utility/uncertainty.py @@ -39,13 +39,18 @@ def percentile_bootstrap_ci(values, n_boot=2000, alpha=0.05, seed=42): reported intervals are reproducible across runs of the same evaluation. :param values: iterable of per-case metric values (may contain NaN) - :param n_boot: number of bootstrap resamples - :param alpha: 1 - confidence level (0.05 -> 95% interval) + :param n_boot: number of bootstrap resamples (must be positive) + :param alpha: 1 - confidence level (0.05 -> 95% interval); must be + strictly between 0 and 1 :param seed: seed for the resampling generator; pass None for non-deterministic resampling :return: (lower, upper) bounds of the interval; (nan, nan) when fewer than two non-NaN values are available """ + if n_boot < 1: + raise ValueError("n_boot must be a positive integer, got %r" % (n_boot,)) + if not 0.0 < alpha < 1.0: + raise ValueError("alpha must be strictly between 0 and 1, got %r" % (alpha,)) arr = np.asarray(values, dtype=float) arr = arr[~np.isnan(arr)] if arr.size < 2: @@ -59,27 +64,33 @@ def percentile_bootstrap_ci(values, n_boot=2000, alpha=0.05, seed=42): ) -def stats_with_ci(df, n_boot=2000, alpha=0.05, seed=42): +def stats_with_ci(df, n_boot=2000, alpha=0.05, seed=42, exclude=()): """Summary statistics of a per-case results dataframe with confidence-interval rows for each numeric metric column. Returns the ``df.describe()`` table augmented with two rows, - ``ci95_low`` and ``ci95_high`` (names follow the requested ``alpha``), + ``ci95_low`` and ``ci95_high`` (names follow the requested ``alpha``; + fractional levels are preserved, e.g. ``alpha=0.025`` -> ``ci97.5_low``), holding the percentile-bootstrap interval for each column's mean. :param df: dataframe of per-case metric values (columns = metrics) :param n_boot: number of bootstrap resamples per column :param alpha: 1 - confidence level :param seed: seed for reproducible intervals + :param exclude: column names to keep in the describe() table but skip + when bootstrapping (identifier columns such as ``case``); their CI + cells are left empty :return: describe() dataframe with appended CI rows """ described = df.describe() - level = int(round((1 - alpha) * 100)) - low_name = "ci%d_low" % level - high_name = "ci%d_high" % level + level = (1 - alpha) * 100 + low_name = "ci%g_low" % level + high_name = "ci%g_high" % level lows = {} highs = {} for col in described.columns: + if col in exclude: + continue series = pd.to_numeric(df[col], errors="coerce") low, high = percentile_bootstrap_ci( series.to_numpy(), n_boot=n_boot, alpha=alpha, seed=seed diff --git a/examples/FN_PredictionPQ.nii.gz b/examples/FN_PredictionPQ.nii.gz new file mode 100644 index 0000000000000000000000000000000000000000..96f5e1a6bcced00861739158603b7d42a9a87caa GIT binary patch literal 118 zcmb2|=3oE;mj7LeDGFQ$h8#d>a7b~L8ao^N28U)_W@fuR2PKvG!)!P>)NJw?elo|- ziZEnrv&nTXIlK7JMF9!k#)|?A964$>EN@(xBEebAZspq+7;=@Vm_L+P)-=b2#mzav TgoRp=fS?83{1OVGG76bqQ literal 0 HcmV?d00001 diff --git a/examples/FP_PredictionPQ.nii.gz b/examples/FP_PredictionPQ.nii.gz new file mode 100644 index 0000000000000000000000000000000000000000..5af28dcefd987099f52f91482b5a4ba018296208 GIT binary patch literal 133 zcmb2|=3oE;mj7LeDGFQ$h8#d>a7b~L8ao^N28U)_W@fuR2PKvG!)!P>)NJw?elo|- ziZEnrv&nTXIlK7JMFEK?iXRTjJ#r|KQu3-7x+`m7_czuk>v0B0Pul?xZpMg8=Edx{ k{u`7ZcyaLAyv8!|1cTp+UmV)`6Mr){daYg9z|Ft_0Nr>m`v3p{ literal 0 HcmV?d00001 diff --git a/examples/FP_PredictionPQ.png b/examples/FP_PredictionPQ.png new file mode 100644 index 0000000000000000000000000000000000000000..f0bb30879857190aa9a0f724327a4f5b92df68d0 GIT binary patch literal 89 zcmeAS@N?(olHy`uVBq!ia0vp^LLkfmBp8a9UhD=^3Z5>GAr*6y6C@@!2)g`f4^UuZ kW>yyFTl(K|6Gy2FBZJK`R>tTU>t2u^Pgg&ebxsLQ06wS{DgXcg literal 0 HcmV?d00001 diff --git a/examples/TP_Pred_PredictionPQ.nii.gz b/examples/TP_Pred_PredictionPQ.nii.gz new file mode 100644 index 0000000000000000000000000000000000000000..976845d9e546936d7bcb80077356d7360b803e18 GIT binary patch literal 161 zcmb2|=3oE;mj7LeDGFQ$h8#d>a7b~L8ao^N28U)_W@fuR2PKvG!)!P>)NJw?elo|- ziZEnrv&nTXIqTf;|DfC>j~Ek|X>}@k z?-GcOF#FMIi0mji8UmvsFp@$5y&NFKr*+s6vL9W3)IE?8Ko1ufA0kUT5-^xO#L~2L Q39U^A0LIDvet-)A0LK74<^TWy literal 0 HcmV?d00001 diff --git a/examples/TP_Ref_PredictionPQ.png b/examples/TP_Ref_PredictionPQ.png new file mode 100644 index 0000000000000000000000000000000000000000..65d9b023c8291580496ba1c999547384cc08bd20 GIT binary patch literal 98 zcmeAS@N?(olHy`uVBq!ia0vp^LLkfmBp8a9UhD=^nw~C>Ar*6y6BO7CjQ+F-a2#kk ua!mI~(=u)?7IuN;$O~5mvTX{TzOXaMrZLG}jIGxPY4&vWb6Mw<&;$T(avE;{ literal 0 HcmV?d00001 diff --git a/test/test_utility/test_uncertainty.py b/test/test_utility/test_uncertainty.py index 984dc29..df87f42 100644 --- a/test/test_utility/test_uncertainty.py +++ b/test/test_utility/test_uncertainty.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd +import pytest from numpy.testing import assert_allclose from MetricsReloaded.utility.uncertainty import ( @@ -62,3 +63,37 @@ def test_stats_with_ci_appends_interval_rows(): assert stats.loc["ci95_high", col] > df[col].mean() # describe() content preserved assert_allclose(stats.loc["mean", "dsc"], df["dsc"].mean()) + + +def test_invalid_options_are_rejected(): + values = [0.7, 0.8, 0.9] + with pytest.raises(ValueError): + percentile_bootstrap_ci(values, n_boot=0) + with pytest.raises(ValueError): + percentile_bootstrap_ci(values, alpha=1.1) + with pytest.raises(ValueError): + percentile_bootstrap_ci(values, alpha=0.0) + + +def test_fractional_confidence_level_is_labelled_exactly(): + df = pd.DataFrame({"dsc": [0.7, 0.8, 0.85, 0.9, 0.75]}) + stats = stats_with_ci(df, alpha=0.025) + # a 97.5% interval must not be rounded to "ci98" + assert "ci97.5_low" in stats.index + assert "ci97.5_high" in stats.index + + +def test_identifier_columns_are_excluded_from_bootstrapping(): + df = pd.DataFrame( + { + "case": [1, 2, 3, 4, 5], + "dsc": [0.7, 0.8, 0.85, 0.9, 0.75], + } + ) + stats = stats_with_ci(df, exclude=("case",)) + # the identifier keeps its describe() rows but gets no interval + assert_allclose(stats.loc["mean", "case"], 3.0) + assert np.isnan(stats.loc["ci95_low", "case"]) + assert np.isnan(stats.loc["ci95_high", "case"]) + # metric columns still get one + assert stats.loc["ci95_low", "dsc"] < df["dsc"].mean() From 7150bedecf23fcc210351deada2c08018c9efa75 Mon Sep 17 00:00:00 2001 From: ipezygj Date: Sun, 9 Aug 2026 06:59:50 +0300 Subject: [PATCH 3/4] Remove test-run artifacts accidentally committed --- examples/FN_PredictionPQ.nii.gz | Bin 118 -> 0 bytes examples/FN_PredictionPQ.png | Bin 81 -> 0 bytes examples/FP_PredictionPQ.nii.gz | Bin 133 -> 0 bytes examples/FP_PredictionPQ.png | Bin 89 -> 0 bytes examples/TP_Pred_PredictionPQ.nii.gz | Bin 161 -> 0 bytes examples/TP_Pred_PredictionPQ.png | Bin 97 -> 0 bytes examples/TP_Ref_PredictionPQ.nii.gz | Bin 162 -> 0 bytes examples/TP_Ref_PredictionPQ.png | Bin 98 -> 0 bytes 8 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 examples/FN_PredictionPQ.nii.gz delete mode 100644 examples/FN_PredictionPQ.png delete mode 100644 examples/FP_PredictionPQ.nii.gz delete mode 100644 examples/FP_PredictionPQ.png delete mode 100644 examples/TP_Pred_PredictionPQ.nii.gz delete mode 100644 examples/TP_Pred_PredictionPQ.png delete mode 100644 examples/TP_Ref_PredictionPQ.nii.gz delete mode 100644 examples/TP_Ref_PredictionPQ.png diff --git a/examples/FN_PredictionPQ.nii.gz b/examples/FN_PredictionPQ.nii.gz deleted file mode 100644 index 96f5e1a6bcced00861739158603b7d42a9a87caa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 118 zcmb2|=3oE;mj7LeDGFQ$h8#d>a7b~L8ao^N28U)_W@fuR2PKvG!)!P>)NJw?elo|- ziZEnrv&nTXIlK7JMF9!k#)|?A964$>EN@(xBEebAZspq+7;=@Vm_L+P)-=b2#mzav TgoRp=fS?83{1OVGG76bqQ diff --git a/examples/FP_PredictionPQ.nii.gz b/examples/FP_PredictionPQ.nii.gz deleted file mode 100644 index 5af28dcefd987099f52f91482b5a4ba018296208..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 133 zcmb2|=3oE;mj7LeDGFQ$h8#d>a7b~L8ao^N28U)_W@fuR2PKvG!)!P>)NJw?elo|- ziZEnrv&nTXIlK7JMFEK?iXRTjJ#r|KQu3-7x+`m7_czuk>v0B0Pul?xZpMg8=Edx{ k{u`7ZcyaLAyv8!|1cTp+UmV)`6Mr){daYg9z|Ft_0Nr>m`v3p{ diff --git a/examples/FP_PredictionPQ.png b/examples/FP_PredictionPQ.png deleted file mode 100644 index f0bb30879857190aa9a0f724327a4f5b92df68d0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 89 zcmeAS@N?(olHy`uVBq!ia0vp^LLkfmBp8a9UhD=^3Z5>GAr*6y6C@@!2)g`f4^UuZ kW>yyFTl(K|6Gy2FBZJK`R>tTU>t2u^Pgg&ebxsLQ06wS{DgXcg diff --git a/examples/TP_Pred_PredictionPQ.nii.gz b/examples/TP_Pred_PredictionPQ.nii.gz deleted file mode 100644 index 976845d9e546936d7bcb80077356d7360b803e18..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 161 zcmb2|=3oE;mj7LeDGFQ$h8#d>a7b~L8ao^N28U)_W@fuR2PKvG!)!P>)NJw?elo|- ziZEnrv&nTXIqTf;|DfC>j~Ek|X>}@k z?-GcOF#FMIi0mji8UmvsFp@$5y&NFKr*+s6vL9W3)IE?8Ko1ufA0kUT5-^xO#L~2L Q39U^A0LIDvet-)A0LK74<^TWy diff --git a/examples/TP_Ref_PredictionPQ.png b/examples/TP_Ref_PredictionPQ.png deleted file mode 100644 index 65d9b023c8291580496ba1c999547384cc08bd20..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 98 zcmeAS@N?(olHy`uVBq!ia0vp^LLkfmBp8a9UhD=^nw~C>Ar*6y6BO7CjQ+F-a2#kk ua!mI~(=u)?7IuN;$O~5mvTX{TzOXaMrZLG}jIGxPY4&vWb6Mw<&;$T(avE;{ From 2bcc4a3b35a80b219437b0fecede07b5dea5b99a Mon Sep 17 00:00:00 2001 From: ipezygj Date: Tue, 11 Aug 2026 08:51:42 +0300 Subject: [PATCH 4/4] uncertainty: reject n_boot=1 instead of returning a zero-width interval Both bounds are quantiles of the resample distribution, so a single resample makes them the same number: percentile_bootstrap_ci(values, n_boot=1) returned (0.5, 0.5) on a sample spanning 0.1 to 0.9. The old check only asked whether n_boot was positive, so the input that produces the strongest possible claim -- an interval of exactly zero width -- was also the cheapest one to pass. Requires n_boot >= 2 and documents the sharper limit behind it: while n_boot < 2 / alpha, both bounds are decided by the most extreme draws alone (fewer than 40 resamples at the default alpha=0.05), so the interval comes out too narrow rather than merely noisy. --- MetricsReloaded/utility/uncertainty.py | 14 +++++++++++--- test/test_utility/test_uncertainty.py | 8 ++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/MetricsReloaded/utility/uncertainty.py b/MetricsReloaded/utility/uncertainty.py index 90a019c..ed56618 100644 --- a/MetricsReloaded/utility/uncertainty.py +++ b/MetricsReloaded/utility/uncertainty.py @@ -39,7 +39,11 @@ def percentile_bootstrap_ci(values, n_boot=2000, alpha=0.05, seed=42): reported intervals are reproducible across runs of the same evaluation. :param values: iterable of per-case metric values (may contain NaN) - :param n_boot: number of bootstrap resamples (must be positive) + :param n_boot: number of bootstrap resamples; must be at least 2, and in + practice should be far larger. Both bounds are quantiles of the + resample distribution, so while n_boot < 2 / alpha they are decided by + its most extreme draws alone (fewer than 40 resamples at the default + alpha=0.05) and the interval is too narrow rather than merely noisy. :param alpha: 1 - confidence level (0.05 -> 95% interval); must be strictly between 0 and 1 :param seed: seed for the resampling generator; pass None for @@ -47,8 +51,12 @@ def percentile_bootstrap_ci(values, n_boot=2000, alpha=0.05, seed=42): :return: (lower, upper) bounds of the interval; (nan, nan) when fewer than two non-NaN values are available """ - if n_boot < 1: - raise ValueError("n_boot must be a positive integer, got %r" % (n_boot,)) + # n_boot=1 passes any "is it positive?" check and then returns a + # zero-width interval, because both quantiles of a single resample are + # that resample: a claim of exact precision produced by the least + # evidence the function accepts. + if n_boot < 2: + raise ValueError("n_boot must be at least 2, got %r" % (n_boot,)) if not 0.0 < alpha < 1.0: raise ValueError("alpha must be strictly between 0 and 1, got %r" % (alpha,)) arr = np.asarray(values, dtype=float) diff --git a/test/test_utility/test_uncertainty.py b/test/test_utility/test_uncertainty.py index df87f42..c2f1ade 100644 --- a/test/test_utility/test_uncertainty.py +++ b/test/test_utility/test_uncertainty.py @@ -75,6 +75,14 @@ def test_invalid_options_are_rejected(): percentile_bootstrap_ci(values, alpha=0.0) +def test_single_resample_is_rejected_rather_than_reported_as_exact(): + # Both bounds are quantiles of the resample distribution, so with one + # resample they are the same number and the interval has zero width -- + # the strongest possible claim from the weakest possible evidence. + with pytest.raises(ValueError): + percentile_bootstrap_ci([0.1, 0.5, 0.9, 0.3, 0.7], n_boot=1) + + def test_fractional_confidence_level_is_labelled_exactly(): df = pd.DataFrame({"dsc": [0.7, 0.8, 0.85, 0.9, 0.75]}) stats = stats_with_ci(df, alpha=0.025)