From 5ae3e93aefe1664e51e1224e4892f9ee517f0a17 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 19 Jan 2026 13:33:49 +0100 Subject: [PATCH 1/7] Make CMOR tables configurable through new configuration system --- doc/reference/cmor_tables.rst | 4 +- esmvalcore/cmor/table.py | 594 ++++-- .../CMOR_MP_BC_tot.dat | 0 .../CMOR_MP_CFCl3.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_CH4.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_CO.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_CO2.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_ClOX.dat | 0 .../CMOR_MP_DU_tot.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_N2O.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_NH3.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_NO.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_NO2.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_NOX.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_O3.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_OH.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_S.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_SO2.dat | 0 .../CMOR_MP_SO4mm_tot.dat | 0 .../CMOR_MP_SS_tot.dat | 0 .../{custom => cmip5-custom}/CMOR_agb.dat | 0 .../{custom => cmip5-custom}/CMOR_alb.dat | 0 .../{custom => cmip5-custom}/CMOR_albDiff.dat | 0 .../CMOR_albDiffiTr13.dat | 0 .../{custom => cmip5-custom}/CMOR_amoc.dat | 0 .../{custom => cmip5-custom}/CMOR_asr.dat | 0 .../{custom => cmip5-custom}/CMOR_awhea.dat | 0 .../{custom => cmip5-custom}/CMOR_bdalb.dat | 0 .../{custom => cmip5-custom}/CMOR_bhalb.dat | 0 .../{custom => cmip5-custom}/CMOR_ch4s.dat | 0 .../{custom => cmip5-custom}/CMOR_chlora.dat | 0 .../CMOR_clhmtisccp.dat | 0 .../CMOR_clhtkisccp.dat | 0 .../{custom => cmip5-custom}/CMOR_clisccp.dat | 0 .../CMOR_cllmtisccp.dat | 0 .../CMOR_clltkisccp.dat | 0 .../CMOR_clmmtisccp.dat | 0 .../CMOR_clmtkisccp.dat | 0 .../CMOR_cltStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_co2s.dat | 0 .../CMOR_coordinates.dat | 0 .../{custom => cmip5-custom}/CMOR_ctotal.dat | 0 .../{custom => cmip5-custom}/CMOR_dos.dat | 0 .../CMOR_dosStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_dpn2o.dat | 0 .../{custom => cmip5-custom}/CMOR_et.dat | 0 .../CMOR_etStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_fapar.dat | 0 .../CMOR_gppStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_hfns.dat | 0 .../CMOR_hurStderr.dat | 0 .../CMOR_husStderr.dat | 0 .../CMOR_iwpStderr.dat | 0 .../CMOR_lapserate.dat | 0 .../{custom => cmip5-custom}/CMOR_lvp.dat | 0 .../{custom => cmip5-custom}/CMOR_lwcre.dat | 0 .../CMOR_lweGrace.dat | 0 .../{custom => cmip5-custom}/CMOR_lwp.dat | 0 .../CMOR_lwpStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_n2oflux.dat | 0 .../{custom => cmip5-custom}/CMOR_n2os.dat | 0 .../{custom => cmip5-custom}/CMOR_netcre.dat | 0 .../CMOR_od550aerStderr.dat | 0 .../CMOR_od870aerStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_ohc.dat | 0 .../CMOR_prStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_prl.dat | 0 .../CMOR_prodlnox.dat | 0 .../{custom => cmip5-custom}/CMOR_ptype.dat | 0 .../{custom => cmip5-custom}/CMOR_qep.dat | 0 .../{custom => cmip5-custom}/CMOR_rlns.dat | 0 .../{custom => cmip5-custom}/CMOR_rlnst.dat | 0 .../{custom => cmip5-custom}/CMOR_rlnstcs.dat | 0 .../{custom => cmip5-custom}/CMOR_rlntcs.dat | 0 .../{custom => cmip5-custom}/CMOR_rluscs.dat | 0 .../{custom => cmip5-custom}/CMOR_rlut.dat | 0 .../{custom => cmip5-custom}/CMOR_rlutcs.dat | 0 .../{custom => cmip5-custom}/CMOR_rsns.dat | 0 .../{custom => cmip5-custom}/CMOR_rsnst.dat | 0 .../{custom => cmip5-custom}/CMOR_rsnstcs.dat | 0 .../CMOR_rsnstcsnorm.dat | 0 .../{custom => cmip5-custom}/CMOR_rsnt.dat | 0 .../{custom => cmip5-custom}/CMOR_rsntcs.dat | 0 .../{custom => cmip5-custom}/CMOR_rsut.dat | 0 .../{custom => cmip5-custom}/CMOR_rsutcs.dat | 0 .../{custom => cmip5-custom}/CMOR_rtnt.dat | 0 .../{custom => cmip5-custom}/CMOR_rx1day.dat | 0 .../{custom => cmip5-custom}/CMOR_rx5day.dat | 0 .../CMOR_siextent.dat | 0 .../{custom => cmip5-custom}/CMOR_sispeed.dat | 0 .../{custom => cmip5-custom}/CMOR_sithick.dat | 0 .../{custom => cmip5-custom}/CMOR_sm.dat | 0 .../{custom => cmip5-custom}/CMOR_sm1m.dat | 0 .../CMOR_smStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_soz.dat | 0 .../{custom => cmip5-custom}/CMOR_swcre.dat | 0 .../CMOR_tasConf5.dat | 0 .../CMOR_tasConf95.dat | 0 .../{custom => cmip5-custom}/CMOR_tasa.dat | 0 .../{custom => cmip5-custom}/CMOR_tasaga.dat | 0 .../{custom => cmip5-custom}/CMOR_tcw.dat | 0 .../{custom => cmip5-custom}/CMOR_tnn.dat | 0 .../CMOR_tosStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_toz.dat | 0 .../CMOR_tozStderr.dat | 0 .../CMOR_tro3prof.dat | 0 .../CMOR_tro3profStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_troz.dat | 0 .../{custom => cmip5-custom}/CMOR_tsDay.dat | 0 .../{custom => cmip5-custom}/CMOR_tsLCDay.dat | 0 .../CMOR_tsLCNight.dat | 0 .../CMOR_tsLSSysErrDay.dat | 0 .../CMOR_tsLSSysErrNight.dat | 0 .../CMOR_tsLocalAtmErrDay.dat | 0 .../CMOR_tsLocalAtmErrNight.dat | 0 .../CMOR_tsLocalSfcErrDay.dat | 0 .../CMOR_tsLocalSfcErrNight.dat | 0 .../{custom => cmip5-custom}/CMOR_tsNight.dat | 0 .../CMOR_tsStderr.dat | 0 .../CMOR_tsTotalDay.dat | 0 .../CMOR_tsTotalNight.dat | 0 .../CMOR_tsUnCorErrDay.dat | 0 .../CMOR_tsUnCorErrNight.dat | 0 .../CMOR_tsVarDay.dat | 0 .../CMOR_tsVarNight.dat | 0 .../{custom => cmip5-custom}/CMOR_txx.dat | 0 .../{custom => cmip5-custom}/CMOR_uajet.dat | 0 .../{custom => cmip5-custom}/CMOR_vegfrac.dat | 0 .../{custom => cmip5-custom}/CMOR_xch4.dat | 0 .../{custom => cmip5-custom}/CMOR_xco2.dat | 0 .../tables/cmip6-custom/CMIP6_custom.json | 1649 +++++++++++++++++ .../cmip6-custom/convert-cmip5-to-cmip6.py | 60 + .../{Tables => tables}/CMIP7_aerosol.json | 0 .../cmip7/{Tables => tables}/CMIP7_atmos.json | 0 .../{Tables => tables}/CMIP7_atmosChem.json | 0 .../CMIP7_cell_measures.json | 0 .../{Tables => tables}/CMIP7_coordinate.json | 0 .../CMIP7_formula_terms.json | 0 .../cmip7/{Tables => tables}/CMIP7_grids.json | 0 .../cmip7/{Tables => tables}/CMIP7_land.json | 0 .../{Tables => tables}/CMIP7_landIce.json | 0 .../CMIP7_long_name_overrides.json | 0 .../cmip7/{Tables => tables}/CMIP7_ocean.json | 0 .../{Tables => tables}/CMIP7_ocnBgchem.json | 0 .../{Tables => tables}/CMIP7_seaIce.json | 0 esmvalcore/config/__init__.py | 14 +- esmvalcore/config/_config.py | 35 +- esmvalcore/config/_config_validators.py | 27 +- .../configurations/defaults/cmor_tables.yml | 77 + esmvalcore/dataset.py | 14 +- esmvalcore/io/__init__.py | 4 +- esmvalcore/io/local.py | 3 + .../cmor/_fixes/native6/test_era5.py | 6 - .../integration/cmor/test_read_cmor_tables.py | 105 +- tests/integration/cmor/test_table.py | 76 +- tests/integration/conftest.py | 101 +- tests/integration/io/test_local.py | 13 +- tests/integration/recipe/test_recipe.py | 10 +- tests/unit/config/test_config.py | 20 +- tests/unit/config/test_config_validator.py | 4 +- tests/unit/config/test_data_sources.py | 6 + tests/unit/io/local/test_get_data_sources.py | 14 +- tests/unit/preprocessor/test_configuration.py | 9 + 163 files changed, 2484 insertions(+), 361 deletions(-) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_BC_tot.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_CFCl3.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_CH4.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_CO.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_CO2.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_ClOX.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_DU_tot.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_N2O.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_NH3.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_NO.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_NO2.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_NOX.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_O3.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_OH.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_S.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_SO2.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_SO4mm_tot.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_SS_tot.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_agb.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_alb.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_albDiff.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_albDiffiTr13.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_amoc.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_asr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_awhea.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_bdalb.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_bhalb.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_ch4s.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_chlora.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clhmtisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clhtkisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_cllmtisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clltkisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clmmtisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clmtkisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_cltStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_co2s.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_coordinates.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_ctotal.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_dos.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_dosStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_dpn2o.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_et.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_etStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_fapar.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_gppStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_hfns.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_hurStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_husStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_iwpStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lapserate.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lvp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lwcre.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lweGrace.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lwp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lwpStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_n2oflux.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_n2os.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_netcre.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_od550aerStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_od870aerStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_ohc.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_prStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_prl.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_prodlnox.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_ptype.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_qep.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlns.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlnst.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlnstcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlntcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rluscs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlut.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlutcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsns.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsnst.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsnstcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsnstcsnorm.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsnt.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsntcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsut.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsutcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rtnt.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rx1day.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rx5day.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_siextent.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_sispeed.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_sithick.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_sm.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_sm1m.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_smStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_soz.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_swcre.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tasConf5.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tasConf95.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tasa.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tasaga.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tcw.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tnn.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tosStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_toz.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tozStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tro3prof.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tro3profStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_troz.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLCDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLCNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLSSysErrDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLSSysErrNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLocalAtmErrDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLocalAtmErrNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLocalSfcErrDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLocalSfcErrNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsTotalDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsTotalNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsUnCorErrDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsUnCorErrNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsVarDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsVarNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_txx.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_uajet.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_vegfrac.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_xch4.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_xco2.dat (100%) create mode 100644 esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json create mode 100644 esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_aerosol.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_atmos.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_atmosChem.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_cell_measures.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_coordinate.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_formula_terms.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_grids.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_land.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_landIce.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_long_name_overrides.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_ocean.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_ocnBgchem.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_seaIce.json (100%) create mode 100644 esmvalcore/config/configurations/defaults/cmor_tables.yml diff --git a/doc/reference/cmor_tables.rst b/doc/reference/cmor_tables.rst index b969a938ab..b872817e13 100644 --- a/doc/reference/cmor_tables.rst +++ b/doc/reference/cmor_tables.rst @@ -49,10 +49,10 @@ by an underscore and the ``branding_suffix``. For example, the facets ``project: CMIP7, mip: atmos, short_name: tas, branding_suffix: tavg-h2m-hxy-u`` select one of the near-surface air temperature variables in the CMIP7 atmos table: -.. literalinclude:: ../../esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json +.. literalinclude:: ../../esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json :start-at: "tas_tavg-h2m-hxy-u": { :end-at: }, - :caption: One of the ``tas`` variable definitions in the CMIP7 atmos table at `esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json `__. + :caption: One of the ``tas`` variable definitions in the CMIP7 atmos table at `esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json `__. For other projects, the facet ``branding_suffix`` can also be used to distinguish between variables from the same CMOR table that share the same ``short_name``, diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 71c519497b..f19978ce51 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -7,25 +7,30 @@ from __future__ import annotations import copy -import errno import glob +import importlib import json import logging import os from collections import Counter from functools import lru_cache, total_ordering from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING import yaml from esmvalcore.exceptions import RecipeError -logger = logging.getLogger(__name__) +if TYPE_CHECKING: + from collections.abc import Iterable + from io import TextIOWrapper + + from esmvalcore.config import Config, Session + from esmvalcore.typing import Facets -CMORTable = Union["CMIP3Info", "CMIP5Info", "CMIP6Info", "CustomInfo"] +logger = logging.getLogger(__name__) -CMOR_TABLES: dict[str, CMORTable] = {} +CMOR_TABLES: dict[str, InfoBase] = {} """dict of str, obj: CMOR info objects.""" _CMOR_KEYS = ( @@ -37,18 +42,39 @@ ) -def _update_cmor_facets(facets): +def _get_institutes(project: str, dataset: str) -> list[str]: + """Return the institutes given the dataset name in CMIP6.""" + try: + return CMOR_TABLES[project].institutes[dataset] # type: ignore[attr-defined] + except (KeyError, AttributeError): + return [] + + +def _get_activity( + project: str, + exp: str | list[str], +) -> str | list[str] | None: + """Return the activity given the experiment name in CMIP6.""" + try: + if isinstance(exp, list): + return [CMOR_TABLES[project].activities[value][0] for value in exp] # type: ignore[attr-defined] + return CMOR_TABLES[project].activities[exp][0] # type: ignore[attr-defined] + except (KeyError, AttributeError): + return None + + +def _update_cmor_facets(facets: Facets) -> None: """Update `facets` with information from CMOR table.""" - project = facets["project"] - mip = facets["mip"] - short_name = facets["short_name"] - derive = facets.get("derive", False) + project: str = facets["project"] # type: ignore[assignment] + mip: str = facets["mip"] # type: ignore[assignment] + short_name: str = facets["short_name"] # type: ignore[assignment] + derive: bool = facets.get("derive", False) # type: ignore[assignment] table = CMOR_TABLES.get(project) if table: table_entry = table.get_variable( mip, short_name, - branding_suffix=facets.get("branding_suffix"), + branding_suffix=facets.get("branding_suffix"), # type: ignore[arg-type] derived=derive, ) else: @@ -71,6 +97,14 @@ def _update_cmor_facets(facets): key, facets, ) + if "dataset" in facets and "institute" not in facets: + institute = _get_institutes(project, facets["dataset"]) # type: ignore[arg-type] + if institute: + facets["institute"] = institute + if "exp" in facets and "activity" not in facets: + activity = _get_activity(project, facets["exp"]) # type: ignore[arg-type] + if activity: + facets["activity"] = activity def _get_mips(project: str, short_name: str) -> list[str]: @@ -154,6 +188,21 @@ def get_var_info( ) +def load_cmor_tables(cfg: Config) -> None: + """Load the configured CMOR tables into :data:`esmvalcore.cmor.table.CMOR_TABLES`. + + Parameters + ---------- + cfg: + The configuration. + """ + CMOR_TABLES.clear() + if cfg.get("config_developer_file") is not None: + read_cmor_tables(cfg["config_developer_file"]) + for project in cfg["projects"]: + CMOR_TABLES[project] = get_tables(cfg, project) + + def read_cmor_tables(cfg_developer: Path | None = None) -> None: """Read cmor tables required in the configuration. @@ -182,7 +231,7 @@ def read_cmor_tables(cfg_developer: Path | None = None) -> None: def _read_cmor_tables( cfg_file: Path, mtime: float, # noqa: ARG001 -) -> dict[str, CMORTable]: +) -> dict[str, InfoBase]: """Read cmor tables required in the configuration. Parameters @@ -201,7 +250,7 @@ def _read_cmor_tables( with open(var_alt_names_file, encoding="utf-8") as yfile: alt_names = yaml.safe_load(yfile) - cmor_tables: dict[str, CMORTable] = {} + cmor_tables: dict[str, InfoBase] = {} # Try to infer location for custom tables from config-developer.yml file, # if not possible, use default location @@ -264,6 +313,66 @@ def _read_table(cfg_developer, table, install_dir, custom, alt_names): raise ValueError(msg) +_TABLE_CACHE: dict[str, InfoBase] = {} +"""The CMOR tables are cached for faster access.""" + + +def clear_table_cache() -> None: + """Clear the CMOR table cache.""" + _TABLE_CACHE.clear() + + +def get_tables( + session: Session | Config, + project: str, +) -> InfoBase: + """Get the CMOR tables for a project. + + Parameters + ---------- + session: + The configuration. + project: + The project to load a CMOR table for. + """ + if project not in session["projects"]: + msg = f"Unknown project '{project}', please configure it under 'projects'." + raise ValueError(msg) + + kwargs = ( + session["projects"][project] + .get( + "cmor_table", + { + "type": "esmvalcore.cmor.table.NoInfo", + }, + ) + .copy() + ) + if "type" not in kwargs: + msg = ( + f"Missing CMOR table 'type' in configuration of project {project}. " + f"Current configuration is:\n{yaml.safe_dump(kwargs)}" + ) + raise ValueError(msg) + cache_key = str(kwargs) + if cache_key not in _TABLE_CACHE: + module_name, cls_name = kwargs.pop("type").rsplit(".", 1) + module = importlib.import_module(module_name) + cls = getattr(module, cls_name) + tables = cls(**kwargs) + if not isinstance(tables, InfoBase): + msg = ( + "Expected CMOR tables of type `esmvalcore.cmor.table.InfoBase`, " + f"but your configuration for project '{project}' contains " + f"'{tables}' of type '{type(tables)}'." + ) + raise TypeError(msg) + _TABLE_CACHE[cache_key] = tables + + return _TABLE_CACHE[cache_key] + + class InfoBase: """Base class for all table info classes. @@ -271,26 +380,63 @@ class InfoBase: Parameters ---------- - default: object - Default table to look variables on if not found + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. + + default: + Default table to look variables on if not found. - alt_names: list[list[str]] - List of known alternative names for variables + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the file variable_alt_names.yml will be used. strict: bool If False, will look for a variable in other tables if it can not be - found in the requested one + found in the requested one. + + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + paths: + A list of paths to CMOR tables. If the path is relative and exists in + the ``tables`` directory in :mod:`esmvalcore.cmor`, the version of the + tables shipped with ESMValCore will be used. """ - def __init__(self, default, alt_names, strict): + def __init__( + self, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + paths: Iterable[Path] = (), + ) -> None: + # Configure the paths to the CMOR tables. + builtin_tables_path = Path(__file__).parent / "tables" + paths = tuple(Path(os.path.expandvars(p)).expanduser() for p in paths) + self.paths = tuple( + builtin_tables_path / p + if (builtin_tables_path / p).is_dir() + else p + for p in paths + ) + for path in self.paths: + if not path.is_dir(): + raise NotADirectoryError(path) + + # Configure the alternative names. if alt_names is None: - alt_names = "" - self.default = default + alt_names_path = Path(__file__).parent / "variable_alt_names.yml" + alt_names = yaml.safe_load( + alt_names_path.read_text(encoding="utf-8"), + ) self.alt_names = alt_names + self.coords: dict[str, CoordinateInfo] = {} + self.default = default self.strict = strict - self.tables = {} + self.tables: dict[str, TableInfo] = {} - def get_table(self, table): + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. Parameters @@ -365,15 +511,6 @@ def get_variable( # cmor_strict=False or derived=True var_info = self._look_in_all_tables(derived, alt_names_list) - # If that didn't work either, look in default table if - # cmor_strict=False or derived=True - if not var_info: - var_info = self._look_in_default( - derived, - alt_names_list, - table_name, - ) - # If necessary, adapt frequency of variable (set it to the one from the # requested MIP). E.g., if the user asked for table `Amon`, but the # variable has been found in `day`, use frequency `mon`. @@ -383,16 +520,6 @@ def get_variable( return var_info - def _look_in_default(self, derived, alt_names_list, table_name): - """Look for variable in default table.""" - var_info = None - if not self.strict or derived: - for alt_names in alt_names_list: - var_info = self.default.get_variable(table_name, alt_names) - if var_info: - break - return var_info - def _look_in_all_tables(self, derived, alt_names_list): """Look for variable in all tables.""" var_info = None @@ -439,53 +566,79 @@ class CMIP6Info(InfoBase): Parameters ---------- - cmor_tables_path: str - Path to the folder containing the Tables folder with the json files + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. - default: object - Default table to look variables on if not found + default: + Default table to look variables on if not found. + + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the file variable_alt_names.yml will be used. strict: bool If False, will look for a variable in other tables if it can not be - found in the requested one + found in the requested one. + + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + paths: + A list of paths to CMOR tables. If the path is relative and exists in + the installed copy of the + `esmvalcore/cmor/tables `__ + directory it will be used. """ def __init__( self, - cmor_tables_path, - default=None, - alt_names=None, - strict=True, - default_table_prefix="", - ): - super().__init__(default, alt_names, strict) - cmor_tables_path = self._get_cmor_path(cmor_tables_path) - - self._cmor_folder = os.path.join(cmor_tables_path, "Tables") - if glob.glob(os.path.join(self._cmor_folder, "*_CV.json")): - self._load_controlled_vocabulary() + cmor_tables_path: str | None = None, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + default_table_prefix: str = "", + paths: Iterable[Path] = (), + ) -> None: + if cmor_tables_path is not None: + # Support cmor_tables_path for backward compatibility. + # TODO: remove in v2.16.0 + tables_path = Path(self._get_cmor_path(cmor_tables_path)) + if (tables_path / "tables").exists(): + # Support CMIP7 which uses a lowercase "tables" subdirectory. + cmor_folder = tables_path / "tables" + else: + cmor_folder = tables_path / "Tables" + paths = (*tuple(paths), cmor_folder) + super().__init__(default, alt_names, strict, paths=paths) self.default_table_prefix = default_table_prefix + self.var_to_freq: dict[str, dict[str, str]] = {} + self.activities: dict[str, list[str]] = {} + self.institutes: dict[str, list[str]] = {} - self.var_to_freq = {} - - self._load_coordinates() - for json_file in glob.glob(os.path.join(self._cmor_folder, "*.json")): - if "CV_test" in json_file or "grids" in json_file: - continue - try: - self._load_table(json_file) - except Exception: - msg = f"Exception raised when loading {json_file}" - # Logger may not be ready at this stage - if logger.handlers: - logger.error(msg) - else: - print(msg) # noqa: T201 - raise + for path in self.paths: + if not any(path.glob("*.json")): + msg = f"No CMOR tables found in {path}" + raise ValueError(msg) + self._load_controlled_vocabulary(path) + self._load_coordinates(path) + for json_file in glob.glob(os.path.join(path, "*.json")): + if "CV_test" in json_file or "grids" in json_file: + continue + try: + self._load_table(json_file) + except Exception: + msg = f"Exception raised when loading {json_file}" + # Logger may not be ready at this stage + if logger.handlers: + logger.error(msg) + else: + print(msg) # noqa: T201 + raise @staticmethod - def _get_cmor_path(cmor_tables_path): + def _get_cmor_path(cmor_tables_path: str) -> str: if os.path.isdir(cmor_tables_path): return cmor_tables_path cwd = os.path.dirname(os.path.realpath(__file__)) @@ -500,13 +653,15 @@ def _load_table(self, json_file): raw_data = json.loads(inf.read()) if not self._is_table(raw_data): return - table = TableInfo() header = raw_data["Header"] - table.name = header["table_id"].split(" ")[-1] - self.tables[table.name] = table + table_name = header["table_id"].split(" ")[-1] + if table_name not in self.tables: + table = TableInfo() + table.name = table_name + self.tables[table_name] = table + table = self.tables[table_name] generic_levels = header["generic_levels"].split() - table.frequency = header.get("frequency", "") self.var_to_freq[table.name] = {} for var_name, var_data in raw_data["variable_entry"].items(): @@ -520,7 +675,6 @@ def _load_table(self, json_file): var_freqs = (var.frequency for var in table.values()) table_freq, _ = Counter(var_freqs).most_common(1)[0] table.frequency = table_freq - self.tables[table.name] = table def _assign_dimensions(self, var, generic_levels): for dimension in var.dimensions: @@ -544,10 +698,9 @@ def _assign_dimensions(self, var, generic_levels): var.coordinates[dimension] = coord - def _load_coordinates(self): - self.coords = {} + def _load_coordinates(self, path: Path) -> None: for json_file in glob.glob( - os.path.join(self._cmor_folder, "*coordinate*.json"), + os.path.join(path, "*coordinate*.json"), ): with open(json_file, encoding="utf-8") as inf: table_data = json.loads(inf.read()) @@ -556,11 +709,9 @@ def _load_coordinates(self): coord.read_json(table_data["axis_entry"][coord_name]) self.coords[coord_name] = coord - def _load_controlled_vocabulary(self): - self.activities = {} - self.institutes = {} + def _load_controlled_vocabulary(self, path: Path) -> None: for json_file in glob.glob( - os.path.join(self._cmor_folder, "*_CV.json"), + os.path.join(path, "*_CV.json"), ): with open(json_file, encoding="utf-8") as inf: table_data = json.loads(inf.read()) @@ -606,6 +757,58 @@ def _is_table(table_data): return "Header" in table_data +class Obs4MIPsInfo(CMIP6Info): + """Class to read obs4MIPs-like data request. + + This uses CMOR 3 json format + + Parameters + ---------- + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. + + default: + Default table to look variables on if not found. + + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the file variable_alt_names.yml will be used. + + strict: bool + If False, will look for a variable in other tables if it can not be + found in the requested one. + + paths: + A list of paths to CMOR tables. If the path is relative and exists in + the installed copy of the + `esmvalcore/cmor/tables `__ + directory it will be used. + """ + + def __init__( + self, + cmor_tables_path: str | None = None, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + paths: Iterable[Path] = (), + ) -> None: + super().__init__( + cmor_tables_path=cmor_tables_path, + default=default, + alt_names=alt_names, + strict=strict, + paths=paths, + ) + # Remove the prefix from the table_id. + table_id_prefix = "obs4MIPs_" + for name in list(self.tables): + if name.startswith(table_id_prefix): + table = self.tables.pop(name) + self.tables[name[len(table_id_prefix) :]] = table + + @total_ordering class TableInfo(dict): """Container class for storing a CMOR table.""" @@ -698,6 +901,7 @@ def __init__(self, table_type, short_name): Variable's short name. """ super().__init__() + self.name = short_name self.table_type = table_type self.modeling_realm = [] """Modeling realm""" @@ -729,6 +933,9 @@ def __init__(self, table_type, short_name): self._json_data = None + def __repr__(self) -> str: + return f"{self.__class__.__name__}(name={self.name})" + def copy(self): """Return a shallow copy of VariableInfo. @@ -881,54 +1088,66 @@ class CMIP5Info(InfoBase): Parameters ---------- - cmor_tables_path: str - Path to the folder containing the Tables folder with the json files + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. - default: object - Default table to look variables on if not found + default: + Default table to look variables on if not found. + + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the file variable_alt_names.yml will be used. strict: bool If False, will look for a variable in other tables if it can not be - found in the requested one + found in the requested one. + + paths: + A list of paths to CMOR tables. If the path is relative and exists in + the installed copy of the + `esmvalcore/cmor/tables `__ + directory it will be used. """ def __init__( self, - cmor_tables_path, - default=None, - alt_names=None, - strict=True, - ): - super().__init__(default, alt_names, strict) - cmor_tables_path = self._get_cmor_path(cmor_tables_path) - - self._cmor_folder = os.path.join(cmor_tables_path, "Tables") - if not os.path.isdir(self._cmor_folder): - raise OSError( - errno.ENOTDIR, - "CMOR tables path is not a directory", - self._cmor_folder, - ) - - self.strict = strict - self.tables = {} - self.coords = {} - self._current_table = None - self._last_line_read = None - - for table_file in glob.glob(os.path.join(self._cmor_folder, "*")): - if "_grids" in table_file: - continue - try: - self._load_table(table_file) - except Exception: - msg = f"Exception raised when loading {table_file}" - # Logger may not be ready at this stage - if logger.handlers: - logger.error(msg) - else: - print(msg) # noqa: T201 - raise + cmor_tables_path: str | None = None, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + paths: Iterable[Path] = (), + ) -> None: + if cmor_tables_path is not None: + # Support cmor_tables_path for backward compatibility. + # TODO: remove in v2.16.0 + cmor_tables_path = self._get_cmor_path(cmor_tables_path) + cmor_folder = Path(cmor_tables_path) / "Tables" + paths = (*tuple(paths), cmor_folder) + super().__init__(default, alt_names, strict, paths=paths) + + self._current_table: TextIOWrapper | None = None + self._last_line_read = ("", "") + + for path in self.paths: + for table_file in sorted( + glob.glob(os.path.join(path, "*")), + # Read coordinate files before variable files so we can link the + # variables with the coordinates. + key=lambda filename: "coordinate" not in filename, + ): + if "_grids" in table_file: + continue + try: + self._load_table(table_file) + except Exception: + msg = f"Exception raised when loading {table_file}" + # Logger may not be ready at this stage + if logger.handlers: + logger.error(msg) + else: + print(msg) # noqa: T201 + raise @staticmethod def _get_cmor_path(cmor_tables_path): @@ -937,25 +1156,21 @@ def _get_cmor_path(cmor_tables_path): cwd = os.path.dirname(os.path.realpath(__file__)) return os.path.join(cwd, "tables", cmor_tables_path) - def _load_table(self, table_file, table_name=""): - if table_name and table_name in self.tables: - # special case used for updating a table with custom variable file - table = self.tables[table_name] + def _load_table(self, table_file: str) -> None: + table = self._read_table_file(table_file) + if table.name in self.tables: + self.tables[table.name].update(table) else: - # default case: table name is first line of table file - table = None - - self._read_table_file(table_file, table) + self.tables[table.name] = table - def _read_table_file(self, table_file, table=None): + def _read_table_file(self, table_file: str) -> TableInfo: + table = TableInfo() with open(table_file, encoding="utf-8") as self._current_table: self._read_line() while True: key, value = self._last_line_read if key == "table_id": - table = TableInfo() table.name = value[len("Table ") :] - self.tables[table.name] = table elif key == "frequency": table.frequency = value elif key == "modeling_realm": @@ -973,7 +1188,8 @@ def _read_table_file(self, table_file, table=None): table[value] = self._read_variable(value, table.frequency) continue if not self._read_line(): - return + break + return table def _read_line(self): line = self._current_table.readline() @@ -1047,24 +1263,35 @@ class CMIP3Info(CMIP5Info): Parameters ---------- - cmor_tables_path: str - Path to the folder containing the Tables folder with the json files + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. + + default: + Default table to look variables on if not found. - default: object - Default table to look variables on if not found + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the file variable_alt_names.yml will be used. strict: bool If False, will look for a variable in other tables if it can not be - found in the requested one + found in the requested one. + + paths: + A list of paths to CMOR tables. If the path is relative and exists in + the installed copy of the + `esmvalcore/cmor/tables `__ + directory it will be used. """ - def _read_table_file(self, table_file, table=None): + def _read_table_file(self, table_file: str) -> TableInfo: for dim in ("zlevel",): coord = CoordinateInfo(dim) coord.generic_level = True coord.axis = "Z" self.coords[dim] = coord - super()._read_table_file(table_file, table) + return super()._read_table_file(table_file) def _read_coordinate(self, value): coord = super()._read_coordinate(value) @@ -1075,8 +1302,8 @@ def _read_coordinate(self, value): def _read_variable(self, short_name, frequency): var = super()._read_variable(short_name, frequency) - var.frequency = None - var.modeling_realm = None + var.frequency = "" + var.modeling_realm = [] return var @@ -1096,28 +1323,28 @@ def __init__(self, cmor_tables_path: str | Path | None = None) -> None: """Initialize class member.""" self.coords = {} self.tables = {} - self.var_to_freq: dict[str, dict] = {} + self.var_to_freq: dict[str, dict[str, str]] = {} table = TableInfo() table.name = "custom" self.tables[table.name] = table # First, read default custom tables from repository - self._cmor_folder = self._get_cmor_path("custom") - self._read_table_dir(self._cmor_folder) + self.paths = (Path(self._get_cmor_path("cmip5-custom")),) # Second, if given, update default tables with user-defined custom # tables if cmor_tables_path is not None: - self._user_table_folder = self._get_cmor_path(cmor_tables_path) - if not os.path.isdir(self._user_table_folder): + user_table_folder = Path(self._get_cmor_path(cmor_tables_path)) + if not user_table_folder.is_dir(): msg = ( - f"Custom CMOR tables path {self._user_table_folder} is " + f"Custom CMOR tables path {user_table_folder} is " f"not a directory" ) raise ValueError(msg) - self._read_table_dir(self._user_table_folder) - else: - self._user_table_folder = None + self.paths += (user_table_folder,) + + for path in self.paths: + self._read_table_dir(str(path)) def _read_table_dir(self, table_dir: str) -> None: """Read CMOR tables from directory.""" @@ -1131,7 +1358,7 @@ def _read_table_dir(self, table_dir: str) -> None: if dat_file == coordinates_file: continue try: - self._read_table_file(dat_file) + self._load_table(dat_file) except Exception: msg = f"Exception raised when loading {dat_file}" # Logger may not be ready at this stage @@ -1174,12 +1401,10 @@ def get_variable( """ return self.tables["custom"].get(short_name, None) - def _read_table_file( - self, - table_file: str, - table: TableInfo | None = None, # noqa: ARG002 - ) -> None: + def _read_table_file(self, table_file: str) -> TableInfo: """Read a single table file.""" + table = TableInfo() + table.name = "custom" with open(table_file, encoding="utf-8") as self._current_table: self._read_line() while True: @@ -1194,13 +1419,44 @@ def _read_table_file( self.coords[value] = self._read_coordinate(value) continue elif key == "variable_entry": - self.tables["custom"][value] = self._read_variable( - value, - "", - ) + table[value] = self._read_variable(value, "") continue if not self._read_line(): - return + return table + + +class NoInfo(InfoBase): + """Table that can be used for projects that do not have a CMOR table.""" + + def get_variable( + self, + table_name: str, # noqa: ARG002 + short_name: str, + *, + branding_suffix: str | None = None, # noqa: ARG002 + derived: bool = False, # noqa: ARG002 + ) -> VariableInfo | None: + """Search and return the variable information. + + Parameters + ---------- + table_name: + Table name, i.e., the variable's MIP. + short_name: + Variable's short name. + derived: + Variable is derived. Information retrieval for derived variables + always looks in the default tables (usually, the custom tables) if + variable is not found in the requested table. + + Returns + ------- + VariableInfo | None + `VariableInfo` object for the requested variable if found, ``None`` + otherwise. + + """ + return VariableInfo(table_type="No table", short_name=short_name) # Load the default tables on initializing the module. diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_BC_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_BC_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_BC_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_BC_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CFCl3.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CFCl3.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CFCl3.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CFCl3.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CH4.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CH4.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CH4.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CH4.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CO.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CO.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CO2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CO2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO2.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_ClOX.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_ClOX.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_ClOX.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_ClOX.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_DU_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_DU_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_DU_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_DU_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_N2O.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_N2O.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_N2O.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_N2O.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NH3.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NH3.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NH3.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NH3.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NO.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NO.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NO2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NO2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO2.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NOX.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NOX.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NOX.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NOX.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_O3.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_O3.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_O3.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_O3.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_OH.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_OH.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_OH.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_OH.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_S.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_S.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_S.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_S.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_SO2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_SO2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO2.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_SO4mm_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO4mm_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_SO4mm_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO4mm_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_SS_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SS_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_SS_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SS_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_agb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_agb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_agb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_agb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_alb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_alb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_alb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_alb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_albDiff.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiff.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_albDiff.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiff.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_albDiffiTr13.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiffiTr13.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_albDiffiTr13.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiffiTr13.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_amoc.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_amoc.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_amoc.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_amoc.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_asr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_asr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_asr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_asr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_awhea.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_awhea.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_awhea.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_awhea.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_bdalb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_bdalb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_bdalb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_bdalb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_bhalb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_bhalb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_bhalb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_bhalb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ch4s.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ch4s.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ch4s.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ch4s.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_chlora.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_chlora.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_chlora.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_chlora.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clhmtisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clhmtisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clhmtisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clhmtisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clhtkisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clhtkisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clhtkisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clhtkisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_cllmtisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_cllmtisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_cllmtisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_cllmtisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clltkisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clltkisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clltkisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clltkisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clmmtisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clmmtisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clmmtisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clmmtisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clmtkisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clmtkisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clmtkisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clmtkisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_cltStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_cltStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_cltStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_cltStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_co2s.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_co2s.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_co2s.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_co2s.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_coordinates.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_coordinates.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_coordinates.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_coordinates.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ctotal.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ctotal.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ctotal.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ctotal.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_dos.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_dos.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_dos.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_dos.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_dosStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_dosStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_dosStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_dosStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_dpn2o.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_dpn2o.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_dpn2o.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_dpn2o.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_et.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_et.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_et.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_et.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_etStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_etStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_etStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_etStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_fapar.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_fapar.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_fapar.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_fapar.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_gppStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_gppStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_gppStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_gppStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_hfns.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_hfns.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_hfns.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_hfns.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_hurStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_hurStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_hurStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_hurStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_husStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_husStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_husStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_husStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_iwpStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_iwpStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_iwpStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_iwpStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lapserate.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lapserate.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lapserate.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lapserate.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lvp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lvp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lvp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lvp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lwcre.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lwcre.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lwcre.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lwcre.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lweGrace.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lweGrace.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lweGrace.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lweGrace.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lwp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lwp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lwp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lwp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lwpStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lwpStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lwpStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lwpStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_n2oflux.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_n2oflux.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_n2oflux.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_n2oflux.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_n2os.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_n2os.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_n2os.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_n2os.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_netcre.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_netcre.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_netcre.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_netcre.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_od550aerStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_od550aerStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_od550aerStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_od550aerStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_od870aerStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_od870aerStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_od870aerStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_od870aerStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ohc.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ohc.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ohc.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ohc.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_prStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_prStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_prStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_prStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_prl.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_prl.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_prl.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_prl.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_prodlnox.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_prodlnox.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_prodlnox.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_prodlnox.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ptype.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ptype.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ptype.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ptype.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_qep.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_qep.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_qep.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_qep.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlns.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlns.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlns.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlns.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlnst.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnst.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlnst.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnst.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlnstcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnstcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlnstcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnstcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlntcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlntcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlntcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlntcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rluscs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rluscs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rluscs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rluscs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlut.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlut.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlut.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlut.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlutcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlutcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlutcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlutcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsns.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsns.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsns.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsns.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnst.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnst.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnst.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnst.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnstcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnstcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnstcsnorm.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcsnorm.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnstcsnorm.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcsnorm.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnt.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnt.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnt.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnt.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsntcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsntcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsntcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsntcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsut.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsut.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsut.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsut.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsutcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsutcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsutcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsutcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rtnt.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rtnt.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rtnt.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rtnt.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rx1day.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rx1day.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rx1day.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rx1day.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rx5day.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rx5day.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rx5day.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rx5day.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_siextent.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_siextent.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_siextent.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_siextent.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sispeed.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sispeed.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sispeed.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sispeed.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sithick.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sithick.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sithick.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sithick.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sm.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sm.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sm.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sm.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sm1m.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sm1m.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sm1m.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sm1m.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_smStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_smStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_smStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_smStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_soz.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_soz.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_soz.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_soz.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_swcre.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_swcre.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_swcre.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_swcre.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasConf5.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf5.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasConf5.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf5.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasConf95.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf95.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasConf95.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf95.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasa.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasa.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasa.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasa.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasaga.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasaga.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasaga.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasaga.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tcw.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tcw.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tcw.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tcw.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tnn.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tnn.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tnn.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tnn.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tosStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tosStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tosStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tosStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_toz.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_toz.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_toz.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_toz.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tozStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tozStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tozStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tozStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tro3prof.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3prof.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tro3prof.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3prof.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tro3profStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3profStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tro3profStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3profStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_troz.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_troz.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_troz.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_troz.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLCDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLCDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLCNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLCNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsTotalDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsTotalDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsTotalNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsTotalNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsVarDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsVarDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsVarNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsVarNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_txx.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_txx.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_txx.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_txx.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_uajet.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_uajet.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_uajet.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_uajet.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_vegfrac.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_vegfrac.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_vegfrac.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_vegfrac.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_xch4.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_xch4.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_xch4.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_xch4.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_xco2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_xco2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_xco2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_xco2.dat diff --git a/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json b/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json new file mode 100644 index 0000000000..7628aa821e --- /dev/null +++ b/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json @@ -0,0 +1,1649 @@ +{ + "Header": { + "generic_levels": "olevel", + "table_id": "Table custom" + }, + "variable_entry": { + "MP_BC_tot": { + "comment": "positive mass of BC_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of black carbon (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_BC_tot", + "standard_name": "", + "units": "kg" + }, + "MP_CFCl3": { + "comment": "positive mass of CFCl3", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CFCl3 (CFC-11)", + "modeling_realm": "atmos", + "out_name": "MP_CFCl3", + "standard_name": "", + "units": "kg" + }, + "MP_CH4": { + "comment": "positive mass of CH4 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CH4", + "modeling_realm": "atmos", + "out_name": "MP_CH4", + "standard_name": "", + "units": "kg" + }, + "MP_CO": { + "comment": "positive mass of CO in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CO", + "modeling_realm": "atmos", + "out_name": "MP_CO", + "standard_name": "", + "units": "kg" + }, + "MP_CO2": { + "comment": "positive mass of CO2 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CO2", + "modeling_realm": "atmos", + "out_name": "MP_CO2", + "standard_name": "", + "units": "kg" + }, + "MP_ClOX": { + "comment": "positive mass of ClOX", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of ClOX", + "modeling_realm": "atmos", + "out_name": "MP_ClOX", + "standard_name": "", + "units": "kg" + }, + "MP_DU_tot": { + "comment": "positive mass of DU_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of mineral dust (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_DU_tot", + "standard_name": "", + "units": "kg" + }, + "MP_N2O": { + "comment": "positive mass of N2O in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of N2O", + "modeling_realm": "atmos", + "out_name": "MP_N2O", + "standard_name": "", + "units": "kg" + }, + "MP_NH3": { + "comment": "positive mass of NH3 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NH3", + "modeling_realm": "atmos", + "out_name": "MP_NH3", + "standard_name": "", + "units": "kg" + }, + "MP_NO": { + "comment": "positive mass of NO in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NO", + "modeling_realm": "atmos", + "out_name": "MP_NO", + "standard_name": "", + "units": "kg" + }, + "MP_NO2": { + "comment": "positive mass of NO2 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NO2", + "modeling_realm": "atmos", + "out_name": "MP_NO2", + "standard_name": "", + "units": "kg" + }, + "MP_NOX": { + "comment": "positive mass of NOX in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NOX (NO+NO2)", + "modeling_realm": "atmos", + "out_name": "MP_NOX", + "standard_name": "", + "units": "kg" + }, + "MP_O3": { + "comment": "positive mass of O3 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of O3", + "modeling_realm": "atmos", + "out_name": "MP_O3", + "standard_name": "", + "units": "kg" + }, + "MP_OH": { + "comment": "positive mass of OH in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of OH", + "modeling_realm": "atmos", + "out_name": "MP_OH", + "standard_name": "", + "units": "kg" + }, + "MP_S": { + "comment": "positive mass of S in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of S", + "modeling_realm": "atmos", + "out_name": "MP_S", + "standard_name": "", + "units": "kg" + }, + "MP_SO2": { + "comment": "positive mass of SO2 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of SO2", + "modeling_realm": "atmos", + "out_name": "MP_SO2", + "standard_name": "", + "units": "kg" + }, + "MP_SO4mm_tot": { + "comment": "positive mass of SO4mm_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of aerosol sulfate (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_SO4mm_tot", + "standard_name": "", + "units": "kg" + }, + "MP_SS_tot": { + "comment": "positive mass of SS_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of sea salt (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_SS_tot", + "standard_name": "", + "units": "kg" + }, + "agb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Amount of living biomass (organic matter) stored in vegetation above the soil including stem, stump, branches, bark, seeds and foliage, expressed as dry weight", + "dimensions": "longitude latitude time", + "long_name": "Above-Ground Biomass", + "modeling_realm": "land", + "out_name": "agb", + "standard_name": "", + "type": "real", + "units": "kg m-2", + "valid_min": "0.0" + }, + "alb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "calculated from clear-sky fluxes", + "dimensions": "longitude latitude time", + "long_name": "albedo at the surface", + "modeling_realm": "atmos", + "out_name": "alb", + "standard_name": "", + "type": "real", + "units": "1" + }, + "albDiff": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Difference in surface albedo for a given vegetation cover transition", + "modeling_realm": "atmos", + "out_name": "albDiff", + "standard_name": "", + "type": "real", + "units": "1" + }, + "albDiffiTr13": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Difference in Surface Albedo for Vegetation Cover Transition from Forest to Crops and Grasses", + "modeling_realm": "atmos", + "out_name": "albDiffiTr13", + "standard_name": "", + "type": "real", + "units": "1" + }, + "amoc": { + "cell_measures": "area: areacello", + "cell_methods": "time: mean area: where sea", + "comment": "AMOC at the Rapid array (26.5 N)", + "dimensions": "time", + "long_name": "Atlantic Meridional Overturning Circulation", + "modeling_realm": "ocean", + "out_name": "amoc", + "standard_name": "", + "type": "real", + "units": "kg s-1" + }, + "asr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Absorbed shortwave radiation", + "modeling_realm": "atmos", + "out_name": "asr", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "awhea": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "global mean net surface heat flux over open water", + "dimensions": "longitude latitude time", + "long_name": "Global Mean Net Surface Heat Flux Over Open Water", + "modeling_realm": "atmos", + "out_name": "awhea", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "bdalb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Broadband directional albedo at the surface", + "modeling_realm": "land", + "out_name": "bdalb", + "standard_name": "", + "type": "real", + "units": "1" + }, + "bhalb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Broadband bihemispherical albedo at the surface", + "modeling_realm": "land", + "out_name": "bhalb", + "standard_name": "", + "type": "real", + "units": "1" + }, + "ch4s": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "As ch4, but only at the surface", + "dimensions": "longitude latitude time", + "long_name": "Atmosphere CH4 surface", + "modeling_realm": "atmos", + "out_name": "ch4s", + "standard_name": "mole_fraction_of_methane_in_air", + "type": "real", + "units": "1e-09" + }, + "chlora": { + "cell_measures": "area: areacello", + "cell_methods": "area: mean where sea time: mean", + "comment": "calculated at the surface (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "chlorophyll concentration", + "modeling_realm": "ocean", + "out_name": "chlora", + "standard_name": "", + "type": "real", + "units": "kg m-3" + }, + "clhmtisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP High Level Medium-Thickness Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clhmtisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clhtkisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP high level thick cloud area fraction", + "modeling_realm": "atmos", + "out_name": "clhtkisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude plev19 tau time", + "long_name": "ISCCP Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "cllmtisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP Low Level Medium-Thickness Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "cllmtisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clltkisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP low level thick cloud area fraction", + "modeling_realm": "atmos", + "out_name": "clltkisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clmmtisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP Middle Level Medium-Thickness Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clmmtisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clmtkisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP Middle Level Thick Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clmtkisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "cltStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "for the whole atmospheric column, as seen from the surface or the top of the atmosphere. Include both large-scale and convective cloud.", + "dimensions": "longitude latitude time", + "long_name": "Total Cloud Fraction Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "0.01", + "ok_min_mean_abs": "0", + "out_name": "cltStderr", + "standard_name": "", + "type": "real", + "units": "%", + "valid_max": "0.01", + "valid_min": "0" + }, + "co2s": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "As co2, but only at the surface", + "dimensions": "longitude latitude time", + "long_name": "Atmosphere CO2", + "modeling_realm": "atmos", + "out_name": "co2s", + "standard_name": "mole_fraction_of_carbon_dioxide_in_air", + "type": "real", + "units": "1e-06" + }, + "ctotal": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean", + "dimensions": "longitude latitude time", + "long_name": "Total Carbon Mass in Ecosystem", + "modeling_realm": "land", + "out_name": "ctotal", + "standard_name": "", + "type": "real", + "units": "kg m-2" + }, + "dos": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "(unitless) degree of soil saturation for comparing mass based models with volumetric observations.", + "dimensions": "longitude latitude time", + "long_name": "Degree of Soil Saturation", + "modeling_realm": "land", + "ok_max_mean_abs": "1", + "ok_min_mean_abs": "0", + "out_name": "dos", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "2", + "valid_min": "0" + }, + "dosStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "dimensions": "longitude latitude time", + "long_name": "Degree of Soil Saturation Error", + "modeling_realm": "land", + "out_name": "dosStderr", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "dpn2o": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "Positive values correspond to higher partial pressure in sea water than air.", + "dimensions": "longitude latitude time", + "long_name": "Surface Nitrous Oxide (N2O) Partial Pressure Difference between Sea Water and Air", + "modeling_realm": "ocnBgchem", + "out_name": "dpn2o", + "standard_name": "", + "type": "real", + "units": "Pa" + }, + "et": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Evapotranspiration", + "modeling_realm": "atmos", + "out_name": "et", + "standard_name": "", + "type": "real", + "units": "mm day-1" + }, + "etStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Standard deviation", + "dimensions": "longitude latitude time", + "long_name": "Evapotranspiration Error", + "modeling_realm": "land", + "out_name": "etStderr", + "standard_name": "", + "type": "real", + "units": "mm day-1", + "valid_min": "0" + }, + "fapar": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Fraction of Absorbed Photosynthetically Active Radiation", + "modeling_realm": "land", + "out_name": "fapar", + "standard_name": "", + "type": "real", + "units": "1" + }, + "gppStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Standard deviation calculated based on median absolute deviation", + "dimensions": "longitude latitude time", + "long_name": "Carbon Mass Flux out of Atmosphere due to Gross Primary Production on Land Error", + "modeling_realm": "land", + "out_name": "gppStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1", + "valid_min": "0" + }, + "hfns": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Surface Net Heat Flux", + "modeling_realm": "atmos", + "out_name": "hfns", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "hurStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "The relative humidity with respect to liquid water for T> 0 C, and with respect to ice for T<0 C.", + "dimensions": "longitude latitude plev19 time", + "long_name": "Relative Humidity Error", + "modeling_realm": "atmos", + "out_name": "hurStderr", + "standard_name": "", + "type": "real", + "units": "%" + }, + "husStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude plev19 time", + "long_name": "Specific Humidity Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "0.01041", + "ok_min_mean_abs": "-0.0003539", + "out_name": "husStderr", + "standard_name": "", + "type": "real", + "units": "1", + "valid_max": "0.02841", + "valid_min": "-0.000299" + }, + "iwpStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Condensed Ice Path Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "1.0", + "ok_min_mean_abs": "0.0", + "out_name": "iwpStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2", + "valid_max": "5.0", + "valid_min": "0.0" + }, + "lapserate": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Atmospheric lapse rate calculated as -dT/dz in K per km.", + "dimensions": "longitude latitude plev19 time", + "long_name": "Lapse Rate", + "modeling_realm": "atmos", + "out_name": "lapserate", + "standard_name": "", + "type": "real", + "units": "K km-1" + }, + "lvp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "longitude latitude time", + "long_name": "Latent Heat Release from Precipitation", + "modeling_realm": "atmos", + "out_name": "lvp", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "lwcre": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Longwave Cloud Radiative Effect", + "modeling_realm": "atmos", + "out_name": "lwcre", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "lweGrace": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "liquid water equivalent thickness anomaly", + "dimensions": "longitude latitude time", + "long_name": "Liquid Water Equivalent Thickness Anomaly", + "modeling_realm": "land", + "out_name": "lweGrace", + "standard_name": "", + "type": "real", + "units": "m" + }, + "lwp": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "The total mass of liquid water in cloud per unit area.", + "dimensions": "longitude latitude time", + "long_name": "Liquid Water Path", + "modeling_realm": "aerosol", + "out_name": "lwp", + "standard_name": "atmosphere_mass_content_of_cloud_liquid_water", + "type": "real", + "units": "kg m-2" + }, + "lwpStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Liquid Water Path Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "1.0", + "ok_min_mean_abs": "0.0", + "out_name": "lwpStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2", + "valid_max": "5.0", + "valid_min": "0.0" + }, + "n2oflux": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "Positive flux is into the atmosphere.", + "dimensions": "longitude latitude time", + "long_name": "Surface Upward Ocean to Atmosphere Mole Flux of Nitrous Oxide (N2O)", + "modeling_realm": "ocnBgchem", + "out_name": "n2oflux", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "mol m-2 s-1" + }, + "n2os": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "As n2o, but only at the surface", + "dimensions": "longitude latitude time", + "long_name": "Mole Fraction of N2O at Surface", + "modeling_realm": "atmos", + "out_name": "n2os", + "standard_name": "mole_fraction_of_nitrous_oxide_in_air", + "type": "real", + "units": "mol mol-1" + }, + "netcre": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Net Cloud Radiative Effect", + "modeling_realm": "atmos", + "out_name": "netcre", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "od550aerStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "AOD error from the ambient aerosls (i.e., includes aerosol water). Does not include AOD from stratospheric aerosols if these are prescribed but includes other possible background aerosol types.", + "dimensions": "longitude latitude time", + "long_name": "Ambient Aerosol Optical Thickness at 550 nm Error", + "modeling_realm": "aerosol", + "out_name": "od550aerStderr", + "standard_name": "", + "type": "real", + "units": "1" + }, + "od870aerStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "AOD error from the ambient aerosls (i.e., includes aerosol water). Does not include AOD from stratospheric aerosols if these are prescribed but includes other possible background aerosol types.", + "dimensions": "longitude latitude time", + "long_name": "Ambient Aerosol Optical Thickness at 870 nm Error", + "modeling_realm": "aerosol", + "out_name": "od870aerStderr", + "standard_name": "", + "type": "real", + "units": "1" + }, + "ohc": { + "cell_measures": "volume: volcello", + "cell_methods": "time: mean area: where sea", + "comment": "Heat content", + "dimensions": "longitude latitude time olevel", + "generic_levels": "olevel", + "long_name": "Heat content in grid cell", + "modeling_realm": "ocean", + "out_name": "ohc", + "standard_name": "", + "type": "real", + "units": "J" + }, + "prStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at surface; includes both liquid and solid phases from all types of clouds (both large-scale and convective)", + "dimensions": "longitude latitude time", + "long_name": "Precipitation Standard Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "0.001", + "ok_min_mean_abs": "0", + "out_name": "prStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1", + "valid_max": "0.001", + "valid_min": "0" + }, + "prl": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "large scale precipitation at surface; includes both liquid and solid phases from all types of clouds (both large-scale and convective)", + "dimensions": "longitude latitude time", + "long_name": "Large Scale Precipitation", + "modeling_realm": "atmos", + "out_name": "prl", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1" + }, + "prodlnox": { + "comment": "Production NOX (NO+NO2) by lightning globally integrated", + "dimensions": "time", + "long_name": "Tendency of atmosphere mass content of NOx from lightning", + "modeling_realm": "atmos", + "out_name": "prodlnox", + "standard_name": "", + "type": "real", + "units": "kg s-1" + }, + "ptype": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Description of numerical values can be found in GRIB2 - CODE TABLE 4.201", + "dimensions": "longitude latitude time", + "long_name": "Precipitation type", + "modeling_realm": "atmos", + "ok_max_mean_abs": "255", + "ok_min_mean_abs": "0", + "out_name": "ptype", + "standard_name": "", + "type": "real", + "units": "1", + "valid_max": "255", + "valid_min": "0" + }, + "qep": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "evspsbl-pr", + "dimensions": "longitude latitude time", + "long_name": "Net moisture flux into atmosphere", + "modeling_realm": "atmos", + "out_name": "qep", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1" + }, + "rlns": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "difference between downward and upward thermal radiation at the surface of the Earth", + "dimensions": "longitude latitude time", + "long_name": "Surface Net downward Longwave Radiation", + "modeling_realm": "atmos", + "out_name": "rlns", + "positive": "down", + "standard_name": "surface_net_downward_longwave_flux", + "type": "real", + "units": "W m-2" + }, + "rlnst": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "to surface and outer space", + "dimensions": "longitude latitude time", + "long_name": "Net Atmospheric Longwave Cooling", + "modeling_realm": "atmos", + "out_name": "rlnst", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rlnstcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "to surface and outer space (For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Net Atmospheric Longwave Cooling assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rlnstcs", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rlntcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Longwave Radiation assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rlntcs", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rluscs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Surface Upwelling Clear-Sky Longwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "376.3", + "ok_min_mean_abs": "325.6", + "out_name": "rluscs", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2", + "valid_max": "658", + "valid_min": "43.75" + }, + "rlut": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Longwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "234.4", + "ok_min_mean_abs": "207.4", + "out_name": "rlut", + "positive": "up", + "standard_name": "toa_outgoing_longwave_flux", + "type": "real", + "units": "W m-2", + "valid_max": "383.2", + "valid_min": "67.48" + }, + "rlutcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Clear-Sky Longwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "260.4", + "ok_min_mean_abs": "228.9", + "out_name": "rlutcs", + "positive": "up", + "standard_name": "toa_outgoing_longwave_flux_assuming_clear_sky", + "type": "real", + "units": "W m-2", + "valid_max": "377.5", + "valid_min": "70.59" + }, + "rsns": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "amount of solar radiation that reaches the surface of the Earth minus the amount reflected by the Earth's surface", + "dimensions": "longitude latitude time", + "long_name": "Surface Net downward Shortwave Radiation", + "modeling_realm": "atmos", + "out_name": "rsns", + "positive": "down", + "standard_name": "surface_net_downward_shortwave_flux", + "type": "real", + "units": "W m-2" + }, + "rsnst": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "(For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Heating from Shortwave Absorption", + "modeling_realm": "atmos", + "out_name": "rsnst", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsnstcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "(For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Heating from Shortwave Absorption assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rsnstcs", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsnstcsnorm": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "(For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Heating from Shortwave Absorption assuming clear sky normalized by incoming solar radiation", + "modeling_realm": "atmos", + "out_name": "rsnstcsnorm", + "standard_name": "", + "type": "real", + "units": "%" + }, + "rsnt": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Shortwave Radiation", + "modeling_realm": "atmos", + "out_name": "rsnt", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsntcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Shortwave Radiation assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rsntcs", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsut": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Shortwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "114.1", + "ok_min_mean_abs": "96.72", + "out_name": "rsut", + "positive": "up", + "standard_name": "toa_outgoing_shortwave_flux", + "type": "real", + "units": "W m-2", + "valid_max": "421.9", + "valid_min": "-0.02689" + }, + "rsutcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Clear-Sky Shortwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "73.36", + "ok_min_mean_abs": "54.7", + "out_name": "rsutcs", + "positive": "up", + "standard_name": "toa_outgoing_shortwave_flux_assuming_clear_sky", + "type": "real", + "units": "W m-2", + "valid_max": "444", + "valid_min": "0" + }, + "rtnt": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Total Radiation", + "modeling_realm": "atmos", + "out_name": "rtnt", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rx1day": { + "cell_measures": "area: areacella", + "cell_methods": "time: sum within days time: maximum over days", + "comment": "ETCCDI (Extreme climate change index) annual/monthly maximum 1-day precipitation", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly maximum 1-day precipitation", + "modeling_realm": "ground", + "out_name": "rx1day", + "standard_name": "", + "type": "real", + "units": "mm" + }, + "rx5day": { + "cell_measures": "area: areacella", + "cell_methods": "time: sum within days time: maximum over days", + "comment": "ETCCDI (Extreme climate change index) annual/monthly maximum 5-day precipitation", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly maximum 5-day precipitation", + "modeling_realm": "ground", + "out_name": "rx5day", + "standard_name": "", + "type": "real", + "units": "mm" + }, + "siextent": { + "cell_measures": "area: areacello", + "cell_methods": "area: mean where sea time: mean", + "comment": "", + "dimensions": "longitude latitude time", + "long_name": "Sea Ice Extent", + "modeling_realm": "seaIce", + "out_name": "siextent", + "standard_name": "", + "type": "real", + "units": "m2", + "valid_max": "", + "valid_min": "" + }, + "sispeed": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean where sea_ice (comment: mask=siconc)", + "comment": "Speed of ice (i.e. mean absolute velocity) to account for back-and-forth movement of the ice", + "dimensions": "longitude latitude time", + "long_name": "Sea-ice speed", + "modeling_realm": "seaIce", + "out_name": "sispeed", + "standard_name": "sea_ice_speed", + "type": "real", + "units": "m s-1", + "valid_max": "", + "valid_min": "" + }, + "sithick": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean where sea_ice (comment: mask=siconc)", + "comment": "Actual (floe) thickness of sea ice (NOT volume divided by grid area as was done in CMIP5)", + "dimensions": "longitude latitude time", + "long_name": "Sea Ice Thickness", + "modeling_realm": "seaIce", + "out_name": "sithick", + "standard_name": "sea_ice_thickness", + "type": "real", + "units": "m", + "valid_max": "", + "valid_min": "" + }, + "sm": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "the volume of water in all phases in a thin surface soil layer.", + "dimensions": "longitude latitude time", + "long_name": "Volumetric Moisture in Upper Portion of Soil Column", + "modeling_realm": "land", + "ok_max_mean_abs": "1", + "ok_min_mean_abs": "0", + "out_name": "sm", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1", + "valid_min": "0" + }, + "sm1m": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "the volume of water in all phases in the upper 1 metre of the soil column", + "dimensions": "longitude latitude time", + "long_name": "Volumetric Moisture in Upper 1 Metre of Soil Column", + "modeling_realm": "land", + "ok_max_mean_abs": "1", + "ok_min_mean_abs": "0", + "out_name": "sm", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1", + "valid_min": "0" + }, + "smStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "Error of the volume of water in all phases in a thin surface soil layer.", + "dimensions": "longitude latitude time", + "long_name": "Volumetric Moisture in Upper Portion of Soil Column Error", + "modeling_realm": "land", + "out_name": "smStderr", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "soz": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "stratospheric ozone column calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU. Here, the stratosphere is defined as the region where O3 mole fraction >= 125 ppb.", + "dimensions": "longitude latitude time", + "long_name": "Stratospheric Ozone Column (O3 mole fraction >= 125 ppb)", + "modeling_realm": "atmos", + "out_name": "soz", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "swcre": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Shortwave Cloud Radiative Effect", + "modeling_realm": "atmos", + "out_name": "swcre", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "tasConf5": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Near-Surface Air Temperature Uncertainty Range", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "0", + "out_name": "tasConf5", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.", + "valid_min": "0" + }, + "tasConf95": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Near-Surface Air Temperature Uncertainty Range", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "0", + "out_name": "tasConf95", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.", + "valid_min": "0" + }, + "tasa": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Near-Surface Air Temperature Anomaly", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "-20", + "out_name": "tasa", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.0", + "valid_min": "-20.0" + }, + "tasaga": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Global-mean Near-Surface Air Temperature Anomaly", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "-20", + "out_name": "tasaga", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.0", + "valid_min": "-20.0" + }, + "tcw": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude plev19 time", + "long_name": "Mass Fraction of Cloud Total Water (liquid + ice)", + "modeling_realm": "atmos", + "out_name": "tcw", + "standard_name": "", + "type": "real", + "units": "kg kg-1" + }, + "tnn": { + "cell_measures": "area: areacella", + "cell_methods": "time: minimum", + "comment": "ETCCDI (Extreme climate change index) annual/monthly minimum value of daily minimum temperature", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly minimum value of daily minimum temperature", + "modeling_realm": "ground", + "out_name": "tnn", + "standard_name": "", + "type": "real", + "units": "degrees_C" + }, + "tosStderr": { + "cell_measures": "area: areacello", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "longitude latitude time", + "long_name": "Sea Surface Temperature Error", + "modeling_realm": "ocean", + "ok_max_mean_abs": "", + "ok_min_mean_abs": "0", + "out_name": "tosStderr", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "", + "valid_min": "0" + }, + "toz": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Total ozone column calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU.", + "dimensions": "longitude latitude time", + "long_name": "Total Ozone Column", + "modeling_realm": "atmos", + "out_name": "toz", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "tozStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Total ozone column error calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU.", + "dimensions": "longitude latitude time", + "long_name": "Total Ozone Column Error", + "modeling_realm": "atmos", + "out_name": "tozStderr", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "tro3prof": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "latitude plev19 time", + "long_name": "Ozone Volume Mixing Ratio", + "modeling_realm": "atmos", + "out_name": "tro3prof", + "standard_name": "", + "type": "real", + "units": "1e-9", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "tro3profStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "latitude plev19 time", + "long_name": "Ozone Volume Mixing Ratio Error", + "modeling_realm": "atmos", + "out_name": "tro3profStderr", + "standard_name": "", + "type": "real", + "units": "1e-9", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "troz": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "tropospheric ozone column calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU. Here, the troposphere is defined as the region where O3 mole fraction < 125 ppb.", + "dimensions": "longitude latitude time", + "long_name": "Tropospheric Ozone Column (O3 mole fraction < 125 ppb)", + "modeling_realm": "atmos", + "out_name": "troz", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "tsDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "surface temperture daytime", + "modeling_realm": "atmos", + "out_name": "tsDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "350", + "valid_min": "190", + "var_name": "tsDay" + }, + "tsLCDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land cover class daytime", + "modeling_realm": "atmos", + "out_name": "tsLCDay", + "standard_name": "", + "type": "float", + "units": "", + "valid_max": "255", + "valid_min": "0", + "var_name": "tsLCDay" + }, + "tsLCNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land cover class night time", + "modeling_realm": "atmos", + "out_name": "tsLCNight", + "standard_name": "", + "type": "float", + "units": "", + "valid_max": "255", + "valid_min": "0", + "var_name": "tsLCNight" + }, + "tsLSSysErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time", + "long_name": "uncertainty from large-scale systematic errors daytime", + "modeling_realm": "atmos", + "out_name": "tsLSSysErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLSSysErrDay" + }, + "tsLSSysErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time", + "long_name": "uncertainty from large-scale systematic errors night time", + "modeling_realm": "atmos", + "out_name": "tsLSSysErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLSSysErrNight" + }, + "tsLocalAtmErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on atmospheric scales daytime", + "modeling_realm": "atmos", + "out_name": "tsLocalAtmErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalAtmErrDay" + }, + "tsLocalAtmErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on atmospheric scales night time", + "modeling_realm": "atmos", + "out_name": "tsLocalAtmErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalAtmErrNight" + }, + "tsLocalSfcErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on surface scales daytime", + "modeling_realm": "atmos", + "out_name": "tsLocalSfcErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalSfcErrDay" + }, + "tsLocalSfcErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on surface scales night time", + "modeling_realm": "atmos", + "out_name": "tsLocalSfcErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalSfcErrNight" + }, + "tsNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "surface temperature nighttime", + "modeling_realm": "atmos", + "out_name": "tsNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "350", + "valid_min": "190", + "var_name": "tsNight" + }, + "tsStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "\"\"skin\"\" temperature error (i.e., SST for open ocean)", + "dimensions": "longitude latitude time", + "long_name": "Surface Temperature Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "10", + "ok_min_mean_abs": "0", + "out_name": "tsStderr", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0" + }, + "tsTotalDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "total uncertainty of land surface temperature daytime", + "modeling_realm": "atmos", + "out_name": "tsTotalDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsTotalDay" + }, + "tsTotalNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "total uncertainty of land surface temperature night time", + "modeling_realm": "atmos", + "out_name": "tsTotalNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsTotalNight" + }, + "tsUnCorErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from uncorrelated errors daytime", + "modeling_realm": "atmos", + "out_name": "tsUnCorErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsUnCorErrDay" + }, + "tsUnCorErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from uncorrelated errors night time", + "modeling_realm": "atmos", + "out_name": "tsUnCorErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsUnCorErrNight" + }, + "tsVarDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land surface temperature variance daytime", + "modeling_realm": "atmos", + "out_name": "tsVarDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "100", + "valid_min": "0", + "var_name": "tsVarDay" + }, + "tsVarNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land surface temperature variance night time", + "modeling_realm": "atmos", + "out_name": "tsVarNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "100", + "valid_min": "0", + "var_name": "tsVarNight" + }, + "txx": { + "cell_measures": "area: areacella", + "cell_methods": "time: maximum", + "comment": "ETCCDI (Extreme climate change index) annual/monthly maximum value of daily maximum temperature", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly maximum value of daily maximum temperature", + "modeling_realm": "ground", + "out_name": "txx", + "standard_name": "", + "type": "real", + "units": "degrees_C" + }, + "uajet": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Jet position expressed as latitude of maximum meridional wind speed", + "modeling_realm": "atmos", + "out_name": "uajet", + "standard_name": "", + "type": "real", + "units": "degrees" + }, + "vegfrac": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: where land", + "comment": "Fraction of entire grid cell that is not covered by bare soil.", + "dimensions": "longitude latitude time", + "long_name": "Vegetation Fraction", + "modeling_realm": "land", + "out_name": "vegfrac", + "standard_name": "", + "type": "real", + "units": "%" + }, + "xch4": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Satellite retrieved column-average dry-air mole fraction of atmospheric methane (XCH4)", + "dimensions": "longitude latitude time", + "long_name": "Column-average Dry-air Mole Fraction of Atmospheric Methane", + "modeling_realm": "atmos", + "out_name": "xch4", + "standard_name": "", + "type": "real", + "units": "1" + }, + "xco2": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Satellite retrieved column-average dry-air mole fraction of atmospheric carbon dioxide (XCO2)", + "dimensions": "longitude latitude time", + "long_name": "Column-average Dry-air Mole Fraction of Atmospheric Carbon Dioxide", + "modeling_realm": "atmos", + "out_name": "xco2", + "standard_name": "", + "type": "real", + "units": "1" + } + } +} diff --git a/esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py b/esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py new file mode 100644 index 0000000000..4ef96ebaa4 --- /dev/null +++ b/esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py @@ -0,0 +1,60 @@ +"""Convert CMIP5-style custom CMOR tables to a CMIP6-style custom table. + +Example usage: `python convert-cmip5-to-cmip6.py esmvalcore/cmor/tables/cmip5-custom/*.dat` +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterable + + +def read(file: Path) -> dict[str, str]: + """Read a CMIP5-style custom CMOR table file into a `dict`.""" + result = {} + for line in file.read_text(encoding="utf-8").split("\n"): + if not line or line.startswith("!"): + continue + key, value = [elem.strip() for elem in line.split(":", 1)] + result[key] = value + return result + + +def translate(files: Iterable[Path]) -> dict[str, Any]: + """Read in CMIP5-style custom CMOR table files and return a CMIP6-style custom table.""" + result = { + "Header": { + "table_id": "Table custom", + "generic_levels": "olevel", + }, + "variable_entry": {}, + } + for file in files: + # Skip the coordinates file and use standard CMIP6 coordinates instead. + if "coordinates" in file.name: + continue + variable = read(file) + variable_entry = variable.pop("variable_entry") + # Remove the "SOURCE" key which has no meaning in CMOR tables. + variable.pop("SOURCE", None) + # Some files are missing `out_name`, assume it is the same as the entry. + if "out_name" not in variable: + variable["out_name"] = variable_entry + # Use a CMIP6 pressure levels coordinate. + if "plevs" in variable["dimensions"]: + variable["dimensions"] = variable["dimensions"].replace( + "plevs", + "plev19", + ) + result["variable_entry"][variable_entry] = variable + return result + + +if __name__ == "__main__": + table_files = [Path(p) for p in sys.argv[1:]] + print(json.dumps(translate(table_files), indent=4, sort_keys=True)) # noqa: T201 diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_aerosol.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_aerosol.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_aerosol.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_aerosol.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmosChem.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmosChem.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmosChem.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmosChem.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_cell_measures.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_cell_measures.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_cell_measures.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_cell_measures.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_coordinate.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_coordinate.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_coordinate.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_coordinate.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_formula_terms.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_formula_terms.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_formula_terms.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_formula_terms.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_grids.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_grids.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_grids.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_grids.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_land.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_land.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_land.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_land.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_landIce.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_landIce.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_landIce.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_landIce.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_long_name_overrides.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_long_name_overrides.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_long_name_overrides.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_long_name_overrides.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocean.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocean.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocean.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocean.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocnBgchem.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocnBgchem.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocnBgchem.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocnBgchem.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_seaIce.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_seaIce.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_seaIce.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_seaIce.json diff --git a/esmvalcore/config/__init__.py b/esmvalcore/config/__init__.py index 5d23c6b0e2..333c3e30e1 100644 --- a/esmvalcore/config/__init__.py +++ b/esmvalcore/config/__init__.py @@ -13,10 +13,22 @@ """ -from ._config_object import CFG, Config, Session +import contextlib + +import iris + +from esmvalcore.config._config_object import CFG, Config, Session __all__ = ( "CFG", "Config", "Session", ) + +# Set iris.FUTURE flags +for attr, value in { + "save_split_attrs": True, + "date_microseconds": True, +}.items(): + with contextlib.suppress(AttributeError): + setattr(iris.FUTURE, attr, value) diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index 3003c6c0e6..9d9ff9d6b5 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -3,7 +3,6 @@ from __future__ import annotations import collections.abc -import contextlib import logging import os import warnings @@ -11,10 +10,9 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -import iris import yaml -from esmvalcore.cmor.table import CMOR_TABLES, read_cmor_tables +from esmvalcore.cmor.table import read_cmor_tables from esmvalcore.exceptions import ESMValCoreDeprecationWarning, RecipeError if TYPE_CHECKING: @@ -30,15 +28,6 @@ USER_EXTRA_FACETS = Path.home() / ".esmvaltool" / "extra_facets" -# Set iris.FUTURE flags -for attr, value in { - "save_split_attrs": True, - "date_microseconds": True, -}.items(): - with contextlib.suppress(AttributeError): - setattr(iris.FUTURE, attr, value) - - # TODO: remove in v2.15.0 def _deep_update(dictionary, update): for key, value in update.items(): @@ -134,28 +123,6 @@ def get_project_config(project): raise RecipeError(msg) -def get_institutes(variable): - """Return the institutes given the dataset name in CMIP6.""" - dataset = variable["dataset"] - project = variable["project"] - try: - return CMOR_TABLES[project].institutes[dataset] - except (KeyError, AttributeError): - return [] - - -def get_activity(variable): - """Return the activity given the experiment name in CMIP6.""" - project = variable["project"] - try: - exp = variable["exp"] - if isinstance(exp, list): - return [CMOR_TABLES[project].activities[value][0] for value in exp] - return CMOR_TABLES[project].activities[exp][0] - except (KeyError, AttributeError): - return None - - def get_ignored_warnings(project: FacetValue, step: str) -> None | list: """Get ignored warnings for a given preprocessing step.""" if project not in CFG: diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 272f6f56e0..33a94ff36f 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -7,7 +7,6 @@ import warnings from collections.abc import Iterable from functools import lru_cache, partial -from importlib.resources import files as importlib_files from pathlib import Path from typing import TYPE_CHECKING, Any, Literal @@ -15,6 +14,7 @@ from esmvalcore import __version__ as current_version from esmvalcore.cmor.check import CheckLevels +from esmvalcore.cmor.table import load_cmor_tables from esmvalcore.config._config import TASKSEP, load_config_developer from esmvalcore.exceptions import ( ESMValCoreDeprecationWarning, @@ -285,9 +285,10 @@ def validate_drs(value): def validate_config_developer(value): """Validate and load config developer path.""" path = validate_path_or_none(value) - if path is None: - path = importlib_files("esmvalcore") / "config-developer.yml" - load_config_developer(path) + if path is not None: + # This has the side-effect of updating `esmvalcore.config._config.CFG` + # `esmvalcore.cmor.tables.CMOR_TABLES`. + load_config_developer(path) return path @@ -373,12 +374,29 @@ def validate_extra_facets_dir(value): return validate_pathlist(value) +def validate_cmor_tables(value: dict) -> None: + """Validate the CMOR table configuration.""" + # This has the side-effect of updating `esmvalcore.cmor.tables.CMOR_TABLES`. + # + # Relying on global state is not nice, preferably we should get rid of any + # global state except the defaults for starting a new session in + # `esmvalcore.config.CFG`. + # + # Having side effects when updating an `esmvalcore.config.Session` object + # that changes global state of the `esmvalcore` package is nasty and should + # preferably be avoided. This would require passing around session objects + # instead of relying on global state (e.g. esmvalcore.config.CFG, + # esmvalcore.cmor.tables.CMOR_TABLES). + load_cmor_tables({"projects": value}) # type: ignore[arg-type] + + def validate_projects( value: dict, ) -> dict[str, dict[str, Any]]: """Validate projects mapping.""" mapping = validate_dict(value) options_for_project: dict[str, Callable[[Any], Any]] = { + "cmor_table": validate_dict, "data": validate_dict, # TODO: try to create data sources here "extra_facets": validate_dict, "preprocessor_filename_template": validate_string, @@ -393,6 +411,7 @@ def validate_projects( ) raise ValidationError(msg) from None mapping[project][option] = options_for_project[option](val) + validate_cmor_tables(mapping) return mapping diff --git a/esmvalcore/config/configurations/defaults/cmor_tables.yml b/esmvalcore/config/configurations/defaults/cmor_tables.yml new file mode 100644 index 0000000000..06ec45c667 --- /dev/null +++ b/esmvalcore/config/configurations/defaults/cmor_tables.yml @@ -0,0 +1,77 @@ +# CMOR table configuration. +projects: + # Projects hosted on ESGF. + CMIP7: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip7/tables + - cmip6-custom + CMIP6: + cmor_table: &cmip6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + CMIP5: + cmor_table: &cmip5 + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cmip5/Tables + - cmip5-custom + CMIP3: + cmor_table: + type: esmvalcore.cmor.table.CMIP3Info + paths: + - cmip3/Tables + - cmip5-custom + CORDEX: + cmor_table: + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cordex/Tables + - cmip5-custom + obs4MIPs: + cmor_table: + type: esmvalcore.cmor.table.Obs4MIPsInfo + # Set `strict: false` because the tables are incomplete. + strict: false + paths: + - obs4mips/Tables + - cmip6-custom + ana4MIPs: + # CMOR tables are available at https://github.com/PCMDI/ana4MIPs-cmor-tables/ + # but it is not clear how well these match the data. + cmor_table: + <<: *cmip5 + strict: false + # Observational and reanalysis data that can be read in its native format by ESMValCore. + native6: + cmor_table: &native6 + <<: *cmip6 + strict: false + # Data from selected climate models that can be read in its native format by ESMValCore. + ACCESS: + cmor_table: + <<: *native6 + CESM: + cmor_table: + <<: *native6 + EMAC: + cmor_table: + <<: *native6 + ICON: + cmor_table: + <<: *native6 + IPSLCM: + cmor_table: + <<: *native6 + # Data that has been CMORized by ESMValTool + OBS6: + cmor_table: + <<: *cmip6 + strict: false + OBS: + cmor_table: + <<: *cmip5 + strict: false diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index 41690dd8a9..8a6a20c156 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -24,11 +24,7 @@ _update_cmor_facets, ) from esmvalcore.config import CFG -from esmvalcore.config._config import ( - get_activity, - get_institutes, - load_extra_facets, -) +from esmvalcore.config._config import load_extra_facets from esmvalcore.config._data_sources import _get_data_sources from esmvalcore.exceptions import InputFilesNotFound, RecipeError from esmvalcore.io.local import _dates_to_timerange @@ -756,14 +752,6 @@ def _get_extra_facets(self) -> dict[str, Any]: def _augment_facets(self) -> None: extra_facets = self._get_extra_facets() _augment(self.facets, extra_facets) - if "institute" not in self.facets: - institute = get_institutes(self.facets) - if institute: - self.facets["institute"] = institute - if "activity" not in self.facets: - activity = get_activity(self.facets) - if activity: - self.facets["activity"] = activity _update_cmor_facets(self.facets) if self.facets.get("frequency") == "fx": self.facets.pop("timerange", None) diff --git a/esmvalcore/io/__init__.py b/esmvalcore/io/__init__.py index e115c462f2..5a936d47e0 100644 --- a/esmvalcore/io/__init__.py +++ b/esmvalcore/io/__init__.py @@ -67,8 +67,8 @@ def load_data_sources( If no ``priority`` is configured for a data source, the default priority of 1 is used. - Arguments - --------- + Parameters + ---------- session: The configuration. project: diff --git a/esmvalcore/io/local.py b/esmvalcore/io/local.py index 56d0da90da..aad0d7dccf 100644 --- a/esmvalcore/io/local.py +++ b/esmvalcore/io/local.py @@ -600,6 +600,9 @@ def find_data(self, **facets: FacetValue) -> list[LocalFile]: f" within the requested timerange {facets['timerange']}" ) + if files: + self.debug_info = f"F{self.debug_info[len('No f') :]}" + return files def _path2facets(self, path: Path, add_timerange: bool) -> dict[str, str]: diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index b183bd811b..cb7333fd7c 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -674,8 +674,6 @@ def ptype_cmor_e1hr(): ], attributes={"comment": COMMENT}, ) - cube.coord("latitude").long_name = "latitude" - cube.coord("longitude").long_name = "longitude" return CubeList([cube]) @@ -752,8 +750,6 @@ def rlns_cmor_e1hr(): ], attributes={"comment": COMMENT, "positive": "down"}, ) - cube.coord("latitude").long_name = "latitude" # from custom table - cube.coord("longitude").long_name = "longitude" # from custom table return CubeList([cube]) @@ -984,8 +980,6 @@ def rsns_cmor_e1hr(): ], attributes={"comment": COMMENT, "positive": "down"}, ) - cube.coord("latitude").long_name = "latitude" # from custom table - cube.coord("longitude").long_name = "longitude" # from custom table return CubeList([cube]) diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index 86c90454f4..efd296219b 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -1,12 +1,23 @@ +from __future__ import annotations + from pathlib import Path from textwrap import dedent +from typing import TYPE_CHECKING import pytest import yaml -from esmvalcore.cmor.table import CMOR_TABLES, read_cmor_tables +from esmvalcore.cmor.table import ( + CMOR_TABLES, + VariableInfo, + get_tables, + read_cmor_tables, +) from esmvalcore.cmor.table import __file__ as root +if TYPE_CHECKING: + from esmvalcore.config import Session + def test_read_cmor_tables_raiser(): """Test func raiser.""" @@ -22,32 +33,94 @@ def test_read_cmor_tables(): for project in "CMIP5", "CMIP6": table = CMOR_TABLES[project] - assert ( - Path(table._cmor_folder) == table_path / project.lower() / "Tables" + assert table.paths == ( + table_path / project.lower() / "Tables", + table_path / f"{project.lower()}-custom", ) assert table.strict is True project = "OBS" table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "cmip5" / "Tables" + assert table.paths == ( + table_path / "cmip5" / "Tables", + table_path / "cmip5-custom", + ) assert table.strict is False project = "OBS6" table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "cmip6" / "Tables" + assert table.paths == ( + table_path / "cmip6" / "Tables", + table_path / "cmip6-custom", + ) assert table.strict is False project = "obs4MIPs" table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "obs4mips" / "Tables" + assert table.paths == ( + table_path / "obs4mips" / "Tables", + table_path / "cmip6-custom", + ) assert table.strict is False - project = "custom" - table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "custom" - assert table._user_table_folder is None - assert table.coords - assert table.tables["custom"] + +@pytest.mark.parametrize( + ( + "project", + "mip", + "short_name", + "branding_suffix", + ), + [ + ("CMIP7", "atmos", "tas", "tavg-h2m-hxy-u"), + ("CMIP7", "Amon", "alb", None), # custom derived variable + ("CMIP6", "Amon", "tas", None), + ("CMIP6", "Amon", "alb", None), # custom derived variable + ("CMIP6", "Amon", "ch4", "Clim"), # table entry != short_name + ("CMIP5", "Amon", "tas", None), + ("CMIP5", "Amon", "alb", None), # custom derived variable + ("CMIP3", "A1", "tas", None), + ("CMIP3", "A1", "alb", None), # custom derived variable + ("CORDEX", "mon", "tas", None), + ("CORDEX", "mon", "alb", None), # custom derived variable + ("obs4MIPs", "Amon", "tas", None), + ("obs4MIPs", "Amon", "agb", None), # custom variable + ("obs4MIPs", "Amon", "alb", None), # custom derived variable + ("ana4MIPs", "Amon", "tas", None), + ("native6", "Amon", "tas", None), + ("native6", "Amon", "agb", None), # custom variable + ("native6", "Amon", "alb", None), # custom derived variable + ("ACCESS", "Amon", "tas", None), + ("CESM", "Amon", "tas", None), + ("EMAC", "Amon", "tas", None), + ("ICON", "Amon", "tas", None), + ("IPSLCM", "Amon", "tas", None), + ("OBS6", "Amon", "tas", None), + ("OBS6", "Amon", "agb", None), # custom variable + ("OBS6", "Amon", "alb", None), # custom derived variable + ("OBS", "Amon", "tas", None), + ("OBS", "Amon", "agb", None), # custom variable + ("OBS", "Amon", "alb", None), # custom derived variable + ], +) +def test_get_tables( + session: Session, + project: str, + mip: str, + short_name: str, + branding_suffix: str | None, +) -> None: + info = get_tables(session, project) + assert info.tables + vardef = info.get_variable( + mip, + short_name, + branding_suffix=branding_suffix, + derived=short_name == "alb", + ) + assert isinstance(vardef, VariableInfo) + assert vardef.short_name + assert vardef.units CMOR_NEWVAR_ENTRY = dedent( @@ -125,7 +198,7 @@ def test_read_cmor_tables(): ) -def test_read_custom_cmor_tables(tmp_path): +def test_read_custom_cmor_tables_config_developer(tmp_path): """Test reading of custom CMOR tables.""" (tmp_path / "CMOR_newvarfortesting.dat").write_text(CMOR_NEWVAR_ENTRY) (tmp_path / "CMOR_netcre.dat").write_text(CMOR_NETCRE_ENTRY) @@ -152,10 +225,10 @@ def test_read_custom_cmor_tables(tmp_path): assert "custom" in CMOR_TABLES custom_table = CMOR_TABLES["custom"] - assert custom_table._cmor_folder == str( - Path(root).parent / "tables" / "custom", + assert custom_table.paths == ( + Path(root).parent / "tables" / "cmip5-custom", + tmp_path, ) - assert custom_table._user_table_folder == str(tmp_path) # Make sure that default tables have been read assert "alb" in custom_table.tables["custom"] diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 08f4e76795..44feca13c5 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -12,6 +12,7 @@ CMIP5Info, CMIP6Info, CustomInfo, + Obs4MIPsInfo, _get_branding_suffixes, _get_mips, _update_cmor_facets, @@ -24,6 +25,8 @@ def test_update_cmor_facets(): "project": "CMIP6", "mip": "Amon", "short_name": "tas", + "dataset": "CanESM5", + "exp": "historical", } _update_cmor_facets(facets) @@ -32,12 +35,18 @@ def test_update_cmor_facets(): "project": "CMIP6", "mip": "Amon", "short_name": "tas", + "dataset": "CanESM5", "original_short_name": "tas", "standard_name": "air_temperature", "long_name": "Near-Surface Air Temperature", "units": "K", "modeling_realm": ["atmos"], "frequency": "mon", + "activity": "CMIP", + "exp": "historical", + "institute": [ + "CCCma", + ], } assert facets == expected @@ -76,12 +85,9 @@ def setUpClass(cls): We read CMIP6Info once to keep tests times manageable """ cls.variables_info = CMIP6Info( - "cmip6", - default=CustomInfo(), - strict=True, - alt_names=[ - ["sic", "siconc"], - ["tro3", "o3"], + paths=[ + Path("cmip6/Tables"), + Path("cmip6-custom"), ], ) @@ -177,11 +183,12 @@ def setUpClass(cls): We read CMIP6Info once to keep tests times manageable """ - cls.variables_info = CMIP6Info( - cmor_tables_path="obs4mips", - default=CustomInfo(), + cls.variables_info = Obs4MIPsInfo( + paths=[ + Path("obs4mips/Tables"), + Path("cmip6-custom"), + ], strict=False, - default_table_prefix="obs4MIPs_", ) def setUp(self): @@ -190,7 +197,7 @@ def setUp(self): def test_get_table_frequency(self): """Test get table frequency.""" self.assertEqual( - self.variables_info.get_table("obs4MIPs_monStderr").frequency, + self.variables_info.get_table("monStderr").frequency, "mon", ) @@ -215,7 +222,7 @@ def test_get_variable_ndvistderr(self): def test_get_variable_hus(self): """Get hus variable.""" - var = self.variables_info.get_variable("obs4MIPs_Amon", "hus") + var = self.variables_info.get_variable("Amon", "hus") self.assertEqual(var.short_name, "hus") self.assertEqual(var.frequency, "mon") @@ -231,7 +238,7 @@ def test_get_variable_from_custom(self): Note table name obs4MIPs_[mip] """ var = self.variables_info.get_variable( - "obs4MIPs_monStderr", + "monStderr", "prStderr", ) self.assertEqual(var.short_name, "prStderr") @@ -240,7 +247,7 @@ def test_get_variable_from_custom(self): def test_get_variable_from_custom_deriving(self): """Get a variable from default.""" var = self.variables_info.get_variable( - "obs4MIPs_Amon", + "Amon", "swcre", derived=True, ) @@ -248,7 +255,7 @@ def test_get_variable_from_custom_deriving(self): self.assertEqual(var.frequency, "mon") var = self.variables_info.get_variable( - "obs4MIPs_Aday", + "Aday", "swcre", derived=True, ) @@ -269,7 +276,13 @@ def setUpClass(cls): We read CMIP5Info once to keep testing times manageable """ - cls.variables_info = CMIP5Info("cmip5", CustomInfo(), strict=True) + cls.variables_info = CMIP5Info( + paths=[ + Path("cmip5/Tables"), + Path("cmip5-custom"), + ], + strict=True, + ) def setUp(self): self.variables_info.strict = True @@ -356,7 +369,13 @@ def setUpClass(cls): We read CMIP5Info once to keep testing times manageable """ - cls.variables_info = CMIP3Info("cmip3", CustomInfo(), strict=True) + cls.variables_info = CMIP3Info( + paths=[ + Path("cmip3/Tables"), + Path("cmip5-custom"), + ], + strict=True, + ) def setUp(self): self.variables_info.strict = True @@ -443,7 +462,12 @@ def setUpClass(cls): We read CORDEX once to keep testing times manageable """ - cls.variables_info = CMIP5Info("cordex", default=CustomInfo()) + cls.variables_info = CMIP5Info( + paths=[ + Path("cordex/Tables"), + Path("cmip5-custom"), + ], + ) def test_custom_tables_location(self): """Test constructor with custom tables location.""" @@ -478,23 +502,29 @@ def test_custom_tables_default_location(self): expected_cmor_folder = os.path.join( os.path.dirname(esmvalcore.cmor.__file__), "tables", - "custom", + "cmip5-custom", ) - self.assertEqual(custom_info._cmor_folder, expected_cmor_folder) + assert custom_info.paths == (Path(expected_cmor_folder),) self.assertTrue(custom_info.tables["custom"]) self.assertTrue(custom_info.coords) def test_custom_tables_location(self): """Test constructor with custom tables location.""" cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__)) - default_cmor_tables_path = os.path.join(cmor_path, "tables", "custom") + default_cmor_tables_path = os.path.join( + cmor_path, + "tables", + "cmip5-custom", + ) cmor_tables_path = os.path.join(cmor_path, "tables", "cmip5") cmor_tables_path = os.path.abspath(cmor_tables_path) custom_info = CustomInfo(cmor_tables_path) - self.assertEqual(custom_info._cmor_folder, default_cmor_tables_path) - self.assertEqual(custom_info._user_table_folder, cmor_tables_path) + assert custom_info.paths == ( + Path(default_cmor_tables_path), + Path(cmor_tables_path), + ) self.assertTrue(custom_info.tables["custom"]) self.assertTrue(custom_info.coords) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3fae8cd8e6..ff8ec4fdc7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,22 +1,20 @@ from __future__ import annotations import os -from pathlib import Path from typing import TYPE_CHECKING import iris import pytest -import esmvalcore.io.local from esmvalcore.io.local import ( + LocalDataSource, LocalFile, - _replace_tags, _select_files, ) -from esmvalcore.local import _select_drs if TYPE_CHECKING: from collections.abc import Callable, Iterator + from pathlib import Path from esmvalcore.typing import Facets, FacetValue @@ -36,11 +34,12 @@ def create_test_file(filename, tracking_id=None): def _get_files( # noqa: C901,PLR0912 + self: LocalDataSource, root_path: Path, facets: Facets, tracking_id: Iterator[int], suffix: str = "nc", -) -> tuple[list[LocalFile], list[Path]]: +) -> list[LocalFile]: """Return dummy files. Wildcards are only supported for `dataset` and `institute`; in this case @@ -55,34 +54,15 @@ def _get_files( # noqa: C901,PLR0912 else: all_facets = [facets] - # Globs without expanded facets - dir_template = _select_drs("input_dir", facets["project"], "default") # type: ignore[arg-type] - file_template = _select_drs("input_file", facets["project"], "default") # type: ignore[arg-type] - dir_globs = _replace_tags(dir_template, facets) - file_globs = _replace_tags(file_template, facets) - globs = sorted( - root_path / "input" / d / f for d in dir_globs for f in file_globs - ) + self.rootpath = root_path / "input" + facets = dict(facets) + if "original_short_name" in facets: + facets["short_name"] = facets["original_short_name"] files = [] for expanded_facets in all_facets: filenames = [] - dir_template = _select_drs( - "input_dir", - expanded_facets["project"], # type: ignore[arg-type] - "default", - ) - file_template = _select_drs( - "input_file", - expanded_facets["project"], # type: ignore[arg-type] - "default", - ) - - dir_globs = _replace_tags(dir_template, expanded_facets) - file_globs = _replace_tags(file_template, expanded_facets) - filename = str( - root_path / "input" / dir_globs[0] / Path(file_globs[0]).name, - ) + filename = str(self._get_glob_patterns(**expanded_facets)[0]) if filename.endswith("nc"): filename = f"{filename[:-2]}{suffix}" @@ -120,7 +100,7 @@ def _get_files( # noqa: C901,PLR0912 if "timerange" in facets: files = _select_files(files, facets["timerange"]) - return files, globs + return files def _tracking_ids(i=0): @@ -129,37 +109,28 @@ def _tracking_ids(i=0): i += 1 -def _get_find_files_func( +def _get_find_data_func( path: Path, suffix: str = "nc", -) -> Callable[ - ..., - tuple[list[LocalFile], list[Path]] | list[LocalFile], -]: +) -> Callable[..., list[LocalFile]]: tracking_id = _tracking_ids() - def find_files( - self: esmvalcore.io.local.LocalDataSource, - *, - debug: bool = False, + def find_data( + self: LocalDataSource, **facets: FacetValue, - ) -> tuple[list[LocalFile], list[Path]] | list[LocalFile]: - files, file_globs = _get_files(path, facets, tracking_id, suffix) - if debug: - return files, file_globs - return files + ) -> list[LocalFile]: + return _get_files(self, path, facets, tracking_id, suffix) - return find_files + return find_data @pytest.fixture -def patched_datafinder(tmp_path: Path, monkeypatch: pytest.MonkeyPath) -> None: - find_files = _get_find_files_func(tmp_path) - monkeypatch.setattr( - esmvalcore.io.local.LocalDataSource, - "find_data", - find_files, - ) +def patched_datafinder( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + find_data = _get_find_data_func(tmp_path) + monkeypatch.setattr(LocalDataSource, "find_data", find_data) @pytest.fixture @@ -167,12 +138,8 @@ def patched_datafinder_grib( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - find_files = _get_find_files_func(tmp_path, suffix="grib") - monkeypatch.setattr( - esmvalcore.io.local.LocalDataSource, - "find_data", - find_files, - ) + find_data = _get_find_data_func(tmp_path, suffix="grib") + monkeypatch.setattr(LocalDataSource, "find_data", find_data) @pytest.fixture @@ -191,25 +158,17 @@ def patched_failing_datafinder( """ tracking_id = _tracking_ids() - def find_files( - self: esmvalcore.io.local.LocalDataSource, - *, - debug: bool = False, + def find_data( + self: LocalDataSource, **facets: FacetValue, - ) -> tuple[list[LocalFile], list[Path]] | list[LocalFile]: - files, file_globs = _get_files(tmp_path, facets, tracking_id) + ) -> list[LocalFile]: + files = _get_files(self, tmp_path, facets, tracking_id) if facets["frequency"] == "fx": files = [] returned_files = [] for file in files: if not ("AAA" in file.name and "rsutcs" in file.name): returned_files.append(file) - if debug: - return returned_files, file_globs return returned_files - monkeypatch.setattr( - esmvalcore.io.local.LocalDataSource, - "find_data", - find_files, - ) + monkeypatch.setattr(LocalDataSource, "find_data", find_data) diff --git a/tests/integration/io/test_local.py b/tests/integration/io/test_local.py index 81a536f196..fc8e5ffcba 100644 --- a/tests/integration/io/test_local.py +++ b/tests/integration/io/test_local.py @@ -9,6 +9,7 @@ import pytest import yaml +import esmvalcore from esmvalcore.config import CFG from esmvalcore.io.local import ( LocalDataSource, @@ -66,8 +67,13 @@ def create_tree(path, filenames=None, symlinks=None): @pytest.mark.parametrize("cfg", CONFIG["get_output_file"]) -def test_get_output_file(cfg): +def test_get_output_file(monkeypatch, cfg): """Test getting output name for preprocessed files.""" + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) output_file = _get_output_file(cfg["variable"], cfg["preproc_dir"]) expected = Path(cfg["output_file"]) assert output_file == expected @@ -95,6 +101,11 @@ def test_find_files(monkeypatch, root, cfg): pprint.pformat(cfg["variable"]), ) project = cfg["variable"]["project"] + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem(CFG, "drs", {project: cfg["drs"]}) monkeypatch.setitem(CFG, "rootpath", {project: root}) create_tree( diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index b87a696387..7799ba522f 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -3294,7 +3294,7 @@ def test_bias_two_refs_with_mmm(tmp_path, patched_datafinder, session): additional_datasets: - {dataset: CanESM5, group: ref, reference_for_bias: true} - {dataset: CESM2, group: ref, reference_for_bias: true} - - {dataset: MPI-ESM-LR, group: notref} + - {dataset: MPI-ESM1-2-LR, group: notref} scripts: null """) @@ -3562,7 +3562,7 @@ def test_distance_metrics_two_refs_with_mmm( additional_datasets: - {dataset: CESM2, ensemble: r1i1p1f1, reference_for_metric: true} - {dataset: CESM2, ensemble: r2i1p1f1, reference_for_metric: true} - - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM1-2-LR} scripts: null """) @@ -3825,9 +3825,9 @@ def test_align_metadata_invalid_project(tmp_path, patched_datafinder, session): """) msg = ( "align_metadata failed: \"No CMOR tables available for project 'ZZZ'. " - "The following tables are available: custom, CMIP7, CMIP6, CMIP5, " - "CMIP3, OBS, OBS6, native6, obs4MIPs, ana4MIPs, EMAC, CORDEX, IPSLCM, " - 'ICON, CESM, ACCESS."' + "The following tables are available: CMIP7, CMIP6, CMIP5, CMIP3, " + "CORDEX, obs4MIPs, ana4MIPs, native6, ACCESS, CESM, EMAC, ICON, IPSLCM, " + 'OBS6, OBS."' ) with pytest.raises(RecipeError) as exc: get_recipe(tmp_path, content, session) diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 035ccbd7f2..0b7aa40d28 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -161,14 +161,14 @@ def test_get_project_config(mocker): def test_load_default_config(cfg_default, monkeypatch): """Test that the default configuration can be loaded.""" - project_cfg = {} - monkeypatch.setattr(_config, "CFG", project_cfg) root_path = importlib_files("esmvalcore") - default_dev_file = root_path / "config-developer.yml" config_dir = root_path / "config" / "configurations" / "defaults" default_project_settings = dask.config.collect( paths=[str(p) for p in config_dir.glob("extra_facets_*.yml")] - + [str(config_dir / "preprocessor_filename_template.yml")], + + [ + str(config_dir / "preprocessor_filename_template.yml"), + str(config_dir / "cmor_tables.yml"), + ], env={}, )["projects"] @@ -178,7 +178,7 @@ def test_load_default_config(cfg_default, monkeypatch): "auxiliary_data_dir": Path.home() / "auxiliary_data", "check_level": CheckLevels.DEFAULT, "compress_netcdf": False, - "config_developer_file": default_dev_file, + "config_developer_file": None, "dask": { "profiles": { "local_threaded": { @@ -261,9 +261,6 @@ def test_load_default_config(cfg_default, monkeypatch): assert getattr(session, path + "_dir") == session.session_dir / path assert session.plot_dir == session.session_dir / "plots" - # Check that projects were configured - assert project_cfg - def test_rootpath_obs4mips_case_correction(monkeypatch): """Test that the name of the obs4MIPs project is correct in rootpath.""" @@ -328,8 +325,13 @@ def test_get_ignored_warnings_none(project, step): assert get_ignored_warnings(project, step) is None -def test_get_ignored_warnings_emac(): +def test_get_ignored_warnings_emac(monkeypatch: pytest.MonkeyPatch) -> None: """Test ``get_ignored_warnings``.""" + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) ignored_warnings = get_ignored_warnings("EMAC", "load") assert isinstance(ignored_warnings, list) assert ignored_warnings diff --git a/tests/unit/config/test_config_validator.py b/tests/unit/config/test_config_validator.py index eed9b19bd5..44e92e86cc 100644 --- a/tests/unit/config/test_config_validator.py +++ b/tests/unit/config/test_config_validator.py @@ -311,13 +311,13 @@ def test_handle_deprecation(remove_version): def test_validate_config_developer_none(): """Test ``validate_config_developer``.""" path = validate_config_developer(None) - assert path == Path(esmvalcore.__file__).parent / "config-developer.yml" + assert path is None def test_validate_config_developer(tmp_path): """Test ``validate_config_developer``.""" custom_table_path = ( - Path(esmvalcore.__file__).parent / "cmor" / "tables" / "custom" + Path(esmvalcore.__file__).parent / "cmor" / "tables" / "cmip5-custom" ) cfg_dev = { "custom": {"cmor_path": custom_table_path}, diff --git a/tests/unit/config/test_data_sources.py b/tests/unit/config/test_data_sources.py index b1e08f69af..fce0b3a625 100644 --- a/tests/unit/config/test_data_sources.py +++ b/tests/unit/config/test_data_sources.py @@ -1,5 +1,6 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING import pytest @@ -37,6 +38,11 @@ def test_load_legacy_data_sources( session["projects"][project].pop("data", None) session["search_esgf"] = search_esgf session["download_dir"] = "~/climate_data" + monkeypatch.setitem( + esmvalcore.local.CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem( esmvalcore.local.CFG, "rootpath", diff --git a/tests/unit/io/local/test_get_data_sources.py b/tests/unit/io/local/test_get_data_sources.py index 8f19709b8a..c086947a57 100644 --- a/tests/unit/io/local/test_get_data_sources.py +++ b/tests/unit/io/local/test_get_data_sources.py @@ -5,8 +5,8 @@ import pytest +import esmvalcore from esmvalcore.config import CFG -from esmvalcore.config._config_validators import validate_config_developer from esmvalcore.io.local import LocalDataSource from esmvalcore.local import DataSource, _get_data_sources @@ -33,7 +33,11 @@ ) def test_get_data_sources(monkeypatch, rootpath_drs): # Make sure that default config-developer file is used - validate_config_developer(None) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) rootpath, drs = rootpath_drs monkeypatch.setitem(CFG, "rootpath", rootpath) @@ -48,7 +52,11 @@ def test_get_data_sources(monkeypatch, rootpath_drs): def test_get_data_sources_nodefault(monkeypatch): # Make sure that default config-developer file is used - validate_config_developer(None) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem( CFG, diff --git a/tests/unit/preprocessor/test_configuration.py b/tests/unit/preprocessor/test_configuration.py index 454b9514a2..539f237873 100644 --- a/tests/unit/preprocessor/test_configuration.py +++ b/tests/unit/preprocessor/test_configuration.py @@ -2,10 +2,13 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING import pytest +import esmvalcore +import esmvalcore.config from esmvalcore.dataset import Dataset from esmvalcore.exceptions import RecipeError from esmvalcore.preprocessor import ( @@ -147,9 +150,15 @@ def test_get_preprocessor_filename_default( def test_get_preprocessor_filename_falls_back_to_config_developer( + monkeypatch: pytest.MonkeyPatch, session: Session, ) -> None: """Test the function `_get_preprocessor_filename`.""" + monkeypatch.setitem( + esmvalcore.config.CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) session["projects"]["CMIP6"].pop("preprocessor_filename_template") dataset = Dataset( project="CMIP6", From aaaeb9e15c3caeee116d257232bea0ac729b6742 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 19 Jan 2026 17:07:30 +0100 Subject: [PATCH 2/7] Fix another test --- tests/integration/io/test_local.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/io/test_local.py b/tests/integration/io/test_local.py index fc8e5ffcba..b15f86ac75 100644 --- a/tests/integration/io/test_local.py +++ b/tests/integration/io/test_local.py @@ -187,7 +187,12 @@ def test_find_data(root, cfg): assert str(pattern) in data_source.debug_info -def test_select_invalid_drs_structure(): +def test_select_invalid_drs_structure(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) msg = ( r"drs _INVALID_STRUCTURE_ for CMIP6 project not specified in " r"config-developer file" From ea4aa6a6eedb36d3f22b460d6607f73f8c620cf3 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 23 Jan 2026 17:49:29 +0100 Subject: [PATCH 3/7] Add docstrings and type hints --- esmvalcore/cmor/_utils.py | 2 +- esmvalcore/cmor/table.py | 270 +++++++++++++++++------- esmvalcore/config/_config.py | 1 + esmvalcore/config/_config_validators.py | 25 +++ 4 files changed, 217 insertions(+), 81 deletions(-) diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py index da8eddd759..f939caadf3 100644 --- a/esmvalcore/cmor/_utils.py +++ b/esmvalcore/cmor/_utils.py @@ -163,7 +163,7 @@ def _get_new_generic_level_coord( New generic level coordinate. """ - new_coord = generic_level_coord.generic_lev_coords[new_coord_name] + new_coord = generic_level_coord.generic_lev_coords[new_coord_name] # type: ignore[index] # Is this a bug? new_coord.generic_level = True new_coord.generic_lev_coords = var_info.coordinates[ generic_level_coord_name diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index f19978ce51..f3610eb49a 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -15,7 +15,7 @@ from collections import Counter from functools import lru_cache, total_ordering from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Self import yaml @@ -43,7 +43,7 @@ def _get_institutes(project: str, dataset: str) -> list[str]: - """Return the institutes given the dataset name in CMIP6.""" + """Return the institutes from the controlled vocabulary given the dataset name.""" try: return CMOR_TABLES[project].institutes[dataset] # type: ignore[attr-defined] except (KeyError, AttributeError): @@ -54,7 +54,7 @@ def _get_activity( project: str, exp: str | list[str], ) -> str | list[str] | None: - """Return the activity given the experiment name in CMIP6.""" + """Return the activity from the controlled vocabulary given the experiment name.""" try: if isinstance(exp, list): return [CMOR_TABLES[project].activities[value][0] for value in exp] # type: ignore[attr-defined] @@ -206,6 +206,12 @@ def load_cmor_tables(cfg: Config) -> None: def read_cmor_tables(cfg_developer: Path | None = None) -> None: """Read cmor tables required in the configuration. + .. deprecated:: 2.14.0 + + The config-developer.yml file based configuration is deprecated and + will no longer be supported in ESMValCore v2.16.0. Please use + :func:`~esmvalcore.cmor.table.load_cmor_tables` instead of this function. + Parameters ---------- cfg_developer: @@ -374,34 +380,34 @@ def get_tables( class InfoBase: - """Base class for all table info classes. - - This uses CMOR 3 json format + """Base class for all CMOR table info classes. Parameters ---------- - cmor_tables_path: - The path to a directory with subdirectory "Tables" where the CMOR tables - are located. - default: Default table to look variables on if not found. + .. deprecated:: 2.14.0 + + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + alt_names: List of known alternative names for variables. If no value is provided, - the default values from the file variable_alt_names.yml will be used. + the default values from the installed copy of + `variable_alt_names.yml `_ + will be used. strict: bool If False, will look for a variable in other tables if it can not be found in the requested one. - default_table_prefix: - If the table_id contains a prefix, it can be specified here. - paths: A list of paths to CMOR tables. If the path is relative and exists in - the ``tables`` directory in :mod:`esmvalcore.cmor`, the version of the - tables shipped with ESMValCore will be used. + the installed copy of the + `esmvalcore/cmor/tables `_ + directory it will be used. """ def __init__( @@ -420,6 +426,7 @@ def __init__( else p for p in paths ) + """A list of paths to CMOR tables.""" for path in self.paths: if not path.is_dir(): raise NotADirectoryError(path) @@ -431,10 +438,24 @@ def __init__( alt_names_path.read_text(encoding="utf-8"), ) self.alt_names = alt_names + """List of known alternative names for variables.""" self.coords: dict[str, CoordinateInfo] = {} + """The coordinates defined in these tables.""" self.default = default + """ + Default table to look variables on if not found. + + .. deprecated:: 2.14.0 + + The ``default`` attribute is deprecated and will be removed in + ESMValCore v2.16.0. + """ self.strict = strict + """If False, will look for a variable in other tables if it can not be + found in the requested one. + """ self.tables: dict[str, TableInfo] = {} + """A mapping from table names to :class:`TableInfo` objects.""" def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. @@ -560,9 +581,9 @@ def _look_all_tables(self, alt_names): class CMIP6Info(InfoBase): - """Class to read CMIP6-like data request. + """Class to read CMIP6-like CMOR tables. - This uses CMOR 3 json format + This class reads CMOR 3 json format tables. Parameters ---------- @@ -570,12 +591,24 @@ class CMIP6Info(InfoBase): The path to a directory with subdirectory "Tables" where the CMOR tables are located. + .. deprecated:: 2.14.0 + + The ``cmor_tables_path`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead. + default: Default table to look variables on if not found. + .. deprecated:: 2.14.0 + + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + alt_names: List of known alternative names for variables. If no value is provided, - the default values from the file variable_alt_names.yml will be used. + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. strict: bool If False, will look for a variable in other tables if it can not be @@ -584,11 +617,15 @@ class CMIP6Info(InfoBase): default_table_prefix: If the table_id contains a prefix, it can be specified here. + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. + paths: A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the - `esmvalcore/cmor/tables `__ - directory it will be used. + the installed copy of the `esmvalcore/cmor/tables`_ directory it will + be used. """ def __init__( @@ -613,9 +650,19 @@ def __init__( super().__init__(default, alt_names, strict, paths=paths) self.default_table_prefix = default_table_prefix + """ + If the table_id contains a prefix, it can be specified here. + + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` attribute is deprecated and will be + removed in ESMValCore v2.16.0. + """ self.var_to_freq: dict[str, dict[str, str]] = {} self.activities: dict[str, list[str]] = {} + """A mapping from ``exp`` to ``activity`` from the controlled vocabulary.""" self.institutes: dict[str, list[str]] = {} + """A mapping from ``dataset`` to ``institute`` from the controlled vocabulary.""" for path in self.paths: if not any(path.glob("*.json")): @@ -665,7 +712,7 @@ def _load_table(self, json_file): self.var_to_freq[table.name] = {} for var_name, var_data in raw_data["variable_entry"].items(): - var = VariableInfo("CMIP6", var_name) + var = VariableInfo("CMIP6", name=var_name) var.read_json(var_data, table.frequency) self._assign_dimensions(var, generic_levels) table[var_name] = var @@ -731,17 +778,17 @@ def _load_controlled_vocabulary(self, path: Path) -> None: except (KeyError, AttributeError): pass - def get_table(self, table): + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. Parameters ---------- - table: str + table: Table name Returns ------- - TableInfo + : Return the TableInfo object for the requested table if found, returns None if not """ @@ -758,22 +805,14 @@ def _is_table(table_data): class Obs4MIPsInfo(CMIP6Info): - """Class to read obs4MIPs-like data request. - - This uses CMOR 3 json format + """Class to read obs4MIPs-like CMOR tables. Parameters ---------- - cmor_tables_path: - The path to a directory with subdirectory "Tables" where the CMOR tables - are located. - - default: - Default table to look variables on if not found. - alt_names: List of known alternative names for variables. If no value is provided, - the default values from the file variable_alt_names.yml will be used. + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. strict: bool If False, will look for a variable in other tables if it can not be @@ -781,22 +820,17 @@ class Obs4MIPsInfo(CMIP6Info): paths: A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the - `esmvalcore/cmor/tables `__ - directory it will be used. + the ``tables`` directory in :mod:`esmvalcore.cmor`, the version of the + tables shipped with ESMValCore will be used. """ def __init__( self, - cmor_tables_path: str | None = None, - default: CustomInfo | None = None, alt_names: list[list[str]] | None = None, strict: bool = True, paths: Iterable[Path] = (), ) -> None: super().__init__( - cmor_tables_path=cmor_tables_path, - default=default, alt_names=alt_names, strict=strict, paths=paths, @@ -817,8 +851,11 @@ def __init__(self, *args, **kwargs): """Create a new TableInfo object for storing VariableInfo objects.""" super().__init__(*args, **kwargs) self.name = "" + """Table name.""" self.frequency = "" + """Table frequency (if defined).""" self.realm = "" + """Table realm (if defined).""" def __eq__(self, other): return (self.name, self.frequency, self.realm) == ( @@ -892,18 +929,38 @@ def _read_json_list_variable(self, parameter): class VariableInfo(JsonInfo): """Class to read and store variable information.""" - def __init__(self, table_type, short_name): + def __init__( + self, + table_type: str = "", + short_name: str = "", + name: str = "", + ) -> None: """Class to read and store variable information. Parameters ---------- - short_name: str + table_type: + Type of table (e.g., CMIP5, CMIP6). + + .. deprecated:: 2.14.0 + + The ``table_type`` parameter is deprecated and will be removed + in ESMValCore v2.16.0. + short_name: Variable's short name. + + .. deprecated:: 2.14.0 + + The ``short_name`` parameter is deprecated and will be removed + in ESMValCore v2.16.0. + name: + Name of the variable entry in the CMOR table. """ super().__init__() - self.name = short_name + self.name = name + """Name of the variable entry in the CMOR table.""" self.table_type = table_type - self.modeling_realm = [] + self.modeling_realm: list[str] = [] """Modeling realm""" self.short_name = short_name """Short name""" @@ -922,9 +979,9 @@ def __init__(self, table_type, short_name): self.positive = "" """Increasing direction""" - self.dimensions = [] + self.dimensions: list[str] = [] """List of dimensions""" - self.coordinates = {} + self.coordinates: dict[str, CoordinateInfo] = {} """Coordinates This is a dict with the names of the dimensions as keys and @@ -936,7 +993,7 @@ def __init__(self, table_type, short_name): def __repr__(self) -> str: return f"{self.__class__.__name__}(name={self.name})" - def copy(self): + def copy(self) -> Self: """Return a shallow copy of VariableInfo. Returns @@ -946,17 +1003,17 @@ def copy(self): """ return copy.copy(self) - def read_json(self, json_data, default_freq): + def read_json(self, json_data: dict, default_freq: str) -> None: """Read variable information from json. Non-present options will be set to empty Parameters ---------- - json_data: dict + json_data: Dictionary created by the json reader containing variable information. - default_freq: str + default_freq: Default frequency to use if it is not defined at variable level. """ self._json_data = json_data @@ -992,12 +1049,12 @@ def has_coord_with_standard_name(self, standard_name: str) -> bool: Parameters ---------- - standard_name: str + standard_name: Standard name to be checked. Returns ------- - bool + : `True` if there is at least one coordinate with the given `standard_name`, `False` if not. @@ -1011,18 +1068,19 @@ def has_coord_with_standard_name(self, standard_name: str) -> bool: class CoordinateInfo(JsonInfo): """Class to read and store coordinate information.""" - def __init__(self, name): + def __init__(self, name: str) -> None: """Class to read and store coordinate information. Parameters ---------- - name: str + name: coordinate's name """ super().__init__() self.name = name + """Name of the coordinate entry in the CMOR table.""" self.generic_level = False - self.generic_lev_coords = {} + self.generic_lev_coords: dict[str, CoordinateInfo] = {} self.axis = "" """Axis""" @@ -1044,7 +1102,7 @@ def __init__(self, name): """Units""" self.stored_direction = "" """Direction in which the coordinate increases""" - self.requested = [] + self.requested: list[str] = [] """Values requested""" self.valid_min = "" """Minimum allowed value""" @@ -1071,7 +1129,7 @@ def read_json(self, json_data): self.axis = self._read_json_variable("axis") self.value = self._read_json_variable("value") self.out_name = self._read_json_variable("out_name") - self.var_name = self._read_json_variable("var_name") + self.var_name = self._read_json_variable("out_name") self.standard_name = self._read_json_variable("standard_name") self.long_name = self._read_json_variable("long_name") self.units = self._read_json_variable("units") @@ -1084,7 +1142,9 @@ def read_json(self, json_data): class CMIP5Info(InfoBase): - """Class to read CMIP5-like data request. + """Class to read CMIP5-like CMOR tables. + + This class reads CMOR 2 format tables. Parameters ---------- @@ -1092,22 +1152,42 @@ class CMIP5Info(InfoBase): The path to a directory with subdirectory "Tables" where the CMOR tables are located. + .. deprecated:: 2.14.0 + + The ``cmor_tables_path`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead. + default: Default table to look variables on if not found. + .. deprecated:: 2.14.0 + + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + alt_names: List of known alternative names for variables. If no value is provided, - the default values from the file variable_alt_names.yml will be used. + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. strict: bool If False, will look for a variable in other tables if it can not be found in the requested one. + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. + paths: A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the - `esmvalcore/cmor/tables `__ - directory it will be used. + the installed copy of the `esmvalcore/cmor/tables`_ directory it will + be used. + """ def __init__( @@ -1224,8 +1304,8 @@ def _read_coordinate(self, value): setattr(coord, key, value) return coord - def _read_variable(self, short_name, frequency): - var = VariableInfo("CMIP5", short_name) + def _read_variable(self, entry_name, frequency): + var = VariableInfo(table_type="CMIP5", name=entry_name) var.frequency = frequency while self._read_line(): key, value = self._last_line_read @@ -1241,17 +1321,17 @@ def _read_variable(self, short_name, frequency): var.coordinates[dim] = self.coords[dim] return var - def get_table(self, table): + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. Parameters ---------- - table: str + table: Table name Returns ------- - TableInfo + : Return the TableInfo object for the requested table if found, returns None if not """ @@ -1259,7 +1339,7 @@ def get_table(self, table): class CMIP3Info(CMIP5Info): - """Class to read CMIP3-like data request. + """Class to read CMIP3-like CMOR tables. Parameters ---------- @@ -1267,22 +1347,42 @@ class CMIP3Info(CMIP5Info): The path to a directory with subdirectory "Tables" where the CMOR tables are located. + .. deprecated:: 2.14.0 + + The ``cmor_tables_path`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead. + default: Default table to look variables on if not found. + .. deprecated:: 2.14.0 + + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + alt_names: List of known alternative names for variables. If no value is provided, - the default values from the file variable_alt_names.yml will be used. + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. strict: bool If False, will look for a variable in other tables if it can not be found in the requested one. + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. + paths: A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the - `esmvalcore/cmor/tables `__ - directory it will be used. + the installed copy of the `esmvalcore/cmor/tables`_ directory it will + be used. + """ def _read_table_file(self, table_file: str) -> TableInfo: @@ -1300,8 +1400,8 @@ def _read_coordinate(self, value): coord.var_name = coord.name return coord - def _read_variable(self, short_name, frequency): - var = super()._read_variable(short_name, frequency) + def _read_variable(self, entry_name, frequency): + var = super()._read_variable(entry_name, frequency) var.frequency = "" var.modeling_realm = [] return var @@ -1310,6 +1410,11 @@ def _read_variable(self, short_name, frequency): class CustomInfo(CMIP5Info): """Class to read custom var info for ESMVal. + .. deprecated:: 2.14.0 + + This class is deprecated and will be removed in ESMValCore v2.16.0. + Please use :class:`~esmvalcore.cmor.tables.table.CMIP5Info` instead. + Parameters ---------- cmor_tables_path: @@ -1426,7 +1531,10 @@ def _read_table_file(self, table_file: str) -> TableInfo: class NoInfo(InfoBase): - """Table that can be used for projects that do not have a CMOR table.""" + """Table that can be used for projects that do not provide a CMOR table.""" + + def __init__(self) -> None: + pass def get_variable( self, @@ -1456,7 +1564,9 @@ def get_variable( otherwise. """ - return VariableInfo(table_type="No table", short_name=short_name) + vardef = VariableInfo(name=short_name) + vardef.short_name = short_name + return vardef # Load the default tables on initializing the module. diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index 9d9ff9d6b5..aaf0ad47c8 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -1,4 +1,5 @@ """Functions dealing with config-developer.yml and extra facets.""" +# TODO: remove this module in v2.16.0 from __future__ import annotations diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 33a94ff36f..6324b374d9 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -630,6 +630,30 @@ def deprecate_search_esgf( ) +def deprecate_config_developer_file( + validated_config: ValidatedConfig, # noqa: ARG001 + value: str | Path, # noqa: ARG001 + validated_value: str | Path, # noqa: ARG001 +) -> None: + """Deprecate ``config_developer_file`` option. + + Parameters + ---------- + validated_config: + ``ValidatedConfig`` instance which will be modified in place. + value: + Raw input value for ``config_file`` option. + validated_value: + Validated value for ``config_file`` option. + + """ + more_info = ( + " Please configure data sources, cmor tables, and preprocessor " + "filename templates under `projects` instead." + ) + _handle_deprecation("config_developer_file", "2.14.0", "2.16.0", more_info) + + # Example usage: see removed files in # https://github.com/ESMValGroup/ESMValCore/pull/2213 _deprecators: dict[str, Callable] = { @@ -638,6 +662,7 @@ def deprecate_search_esgf( "rootpath": deprecate_rootpath, # TODO: remove in v2.16.0 "download_dir": deprecate_download_dir, # TODO: remove in v2.16.0 "search_esgf": deprecate_search_esgf, # TODO: remove in v2.16.0 + "config_developer_file": deprecate_config_developer_file, # TODO: remove in v2.16.0 } From 64f365bb211c3e2c1bc54040b75c11356fe48c87 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 23 Jan 2026 18:03:00 +0100 Subject: [PATCH 4/7] Gracefully handle missing out_name in CMIP5-style CMOR tables and update default config --- esmvalcore/cmor/table.py | 3 +++ esmvalcore/config/_config_validators.py | 1 + esmvalcore/config/configurations/defaults/config-user.yml | 5 ----- tests/unit/cmor/test_table.py | 4 +++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index f3610eb49a..7a083ff58d 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -1317,6 +1317,9 @@ def _read_variable(self, entry_name, frequency): setattr(var, key, value) elif key == "out_name": var.short_name = value + if not var.short_name: + # Some of our custom CMIP5 table entries are missing the `out_name` field. + var.short_name = var.name for dim in var.dimensions: var.coordinates[dim] = self.coords[dim] return var diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 6324b374d9..666400f764 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -671,4 +671,5 @@ def deprecate_config_developer_file( # https://github.com/ESMValGroup/ESMValCore/pull/2213 _deprecated_options_defaults: dict[str, Any] = { "extra_facets_dir": [], # TODO: remove in v2.15.0 + "config_developer_file": None, # TODO: remove in v2.16.0 } diff --git a/esmvalcore/config/configurations/defaults/config-user.yml b/esmvalcore/config/configurations/defaults/config-user.yml index a8b959a70e..bfc7a157c5 100644 --- a/esmvalcore/config/configurations/defaults/config-user.yml +++ b/esmvalcore/config/configurations/defaults/config-user.yml @@ -54,11 +54,6 @@ compress_netcdf: false # step. These files are numbered according to the preprocessing order. save_intermediary_cubes: false -# Path to custom ``config-developer.yml`` file -# This can be used to customise project configurations. See -# ``config-developer.yml`` for an example. Set to ``null`` to use the default. -config_developer_file: null - # Use a profiling tool for the diagnostic run --- [false]/true # A profiler tells you which functions in your code take most time to run. # Only available for Python diagnostics. diff --git a/tests/unit/cmor/test_table.py b/tests/unit/cmor/test_table.py index 155933d958..2a12d18599 100644 --- a/tests/unit/cmor/test_table.py +++ b/tests/unit/cmor/test_table.py @@ -117,8 +117,10 @@ def test_read_standard_name(self): def test_read_var_name(self): """Test var_name.""" + # There does not appear to be a "var_name" field in any of the CMOR + # tables. Could the var_name attribute be removed? info = CoordinateInfo("var") - info.read_json({"var_name": self.value}) + info.read_json({"out_name": self.value}) self.assertEqual(info.var_name, self.value) def test_read_out_name(self): From d676fbd570e756aa4f9a88ff9111dc2a49e17ab3 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 26 Jan 2026 12:16:17 +0100 Subject: [PATCH 5/7] Remove config_developer_file from default configuration --- esmvalcore/config/_config_validators.py | 1 - tests/integration/io/test_local.py | 5 +++++ tests/unit/config/test_config.py | 1 - 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 666400f764..6324b374d9 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -671,5 +671,4 @@ def deprecate_config_developer_file( # https://github.com/ESMValGroup/ESMValCore/pull/2213 _deprecated_options_defaults: dict[str, Any] = { "extra_facets_dir": [], # TODO: remove in v2.15.0 - "config_developer_file": None, # TODO: remove in v2.16.0 } diff --git a/tests/integration/io/test_local.py b/tests/integration/io/test_local.py index b15f86ac75..6818bf85ac 100644 --- a/tests/integration/io/test_local.py +++ b/tests/integration/io/test_local.py @@ -132,6 +132,11 @@ def test_find_files_with_facets(monkeypatch, root): break project = cfg["variable"]["project"] + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem(CFG, "drs", {project: cfg["drs"]}) monkeypatch.setitem(CFG, "rootpath", {project: root}) diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 0b7aa40d28..4c8aed9775 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -178,7 +178,6 @@ def test_load_default_config(cfg_default, monkeypatch): "auxiliary_data_dir": Path.home() / "auxiliary_data", "check_level": CheckLevels.DEFAULT, "compress_netcdf": False, - "config_developer_file": None, "dask": { "profiles": { "local_threaded": { From d0090d48f00daa4b9018a98325ab69f766074b2d Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 26 Jan 2026 12:27:25 +0100 Subject: [PATCH 6/7] Use explicit configuration instead of YAML anchors --- .../configurations/defaults/cmor_tables.yml | 56 +++++++++++++++---- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/esmvalcore/config/configurations/defaults/cmor_tables.yml b/esmvalcore/config/configurations/defaults/cmor_tables.yml index 06ec45c667..5da64dbf6c 100644 --- a/esmvalcore/config/configurations/defaults/cmor_tables.yml +++ b/esmvalcore/config/configurations/defaults/cmor_tables.yml @@ -8,13 +8,13 @@ projects: - cmip7/tables - cmip6-custom CMIP6: - cmor_table: &cmip6 + cmor_table: type: esmvalcore.cmor.table.CMIP6Info paths: - cmip6/Tables - cmip6-custom CMIP5: - cmor_table: &cmip5 + cmor_table: type: esmvalcore.cmor.table.CMIP5Info paths: - cmip5/Tables @@ -43,35 +43,67 @@ projects: # CMOR tables are available at https://github.com/PCMDI/ana4MIPs-cmor-tables/ # but it is not clear how well these match the data. cmor_table: - <<: *cmip5 + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cmip5/Tables + - cmip5-custom strict: false # Observational and reanalysis data that can be read in its native format by ESMValCore. native6: - cmor_table: &native6 - <<: *cmip6 + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom strict: false # Data from selected climate models that can be read in its native format by ESMValCore. ACCESS: cmor_table: - <<: *native6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false CESM: cmor_table: - <<: *native6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false EMAC: cmor_table: - <<: *native6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false ICON: cmor_table: - <<: *native6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false IPSLCM: cmor_table: - <<: *native6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false # Data that has been CMORized by ESMValTool OBS6: cmor_table: - <<: *cmip6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom strict: false OBS: cmor_table: - <<: *cmip5 + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cmip5/Tables + - cmip5-custom strict: false From 72785a7f2a41b25713fbda629796b644995c0068 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 28 Jan 2026 16:03:44 +0100 Subject: [PATCH 7/7] Reduce global state --- esmvalcore/_recipe/check.py | 11 +++++-- esmvalcore/_recipe/recipe.py | 3 +- esmvalcore/cmor/_fixes/fix.py | 39 +++++++++++++++------- esmvalcore/cmor/_utils.py | 34 ++++++++++--------- esmvalcore/cmor/check.py | 49 ++++++++++++++++++++++++--- esmvalcore/cmor/table.py | 53 ++++++++++++++++++++---------- esmvalcore/dataset.py | 2 -- esmvalcore/preprocessor/_other.py | 26 ++++++++++++--- esmvalcore/preprocessor/_regrid.py | 22 ++++++++----- 9 files changed, 172 insertions(+), 67 deletions(-) diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index 1aae4b6aef..4bb2086a05 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -36,6 +36,7 @@ from pathlib import Path from esmvalcore._task import TaskSet + from esmvalcore.config import Session from esmvalcore.dataset import Dataset from esmvalcore.typing import Facets @@ -43,7 +44,7 @@ logger = logging.getLogger(__name__) -def align_metadata(step_settings: dict[str, Any]) -> None: +def align_metadata(step_settings: dict[str, Any], session: Session) -> None: """Check settings of preprocessor ``align_metadata``.""" project = step_settings.get("target_project") mip = step_settings.get("target_mip") @@ -55,7 +56,13 @@ def align_metadata(step_settings: dict[str, Any]) -> None: return try: - _get_var_info(project, mip, short_name) + _get_var_info( + project, + mip, + short_name, + branding_suffix=step_settings.get("branding_suffix"), + session=session, + ) except ValueError as exc: if strict: msg = ( diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 38f48fc663..9fdf9f6685 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -146,6 +146,7 @@ def _update_target_levels( settings["extract_levels"]["levels"] = get_cmor_levels( levels["cmor_table"], levels["coordinate"], + session=dataset.session, ) elif "dataset" in levels: dataset_name = levels["dataset"] @@ -595,7 +596,7 @@ def _update_align_metadata( "target_short_name", dataset.facets["short_name"], ) - check.align_metadata(settings["align_metadata"]) + check.align_metadata(settings["align_metadata"], dataset.session) def _update_extract_shape( diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 26c587f9ed..faf7ac4a3f 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -7,8 +7,9 @@ import inspect import logging import tempfile +import warnings from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Self import dask import numpy as np @@ -26,7 +27,7 @@ _get_single_cube, ) from esmvalcore.cmor.fixes import get_time_bounds -from esmvalcore.cmor.table import get_var_info +from esmvalcore.cmor.table import get_tables from esmvalcore.iris_helpers import ( has_unstructured_grid, safe_convert_units, @@ -53,8 +54,9 @@ class Fix: def __init__( self, vardef: VariableInfo, + *, + session: Session, extra_facets: dict | None = None, - session: Session | None = None, frequency: str | None = None, ) -> None: """Initialize fix object. @@ -63,11 +65,11 @@ def __init__( ---------- vardef: CMOR table entry of the variable. - extra_facets: - Extra facets. For details, see :ref:`config-extra-facets`. session: Current session which includes configuration and directory information. + extra_facets: + Extra facets. For details, see :ref:`config-extra-facets`. frequency: Expected frequency of the variable. If not given, use the one from the CMOR table entry of the variable. @@ -211,7 +213,7 @@ def get_fixes( extra_facets: dict | None = None, session: Session | None = None, frequency: str | None = None, - ) -> list: + ) -> list[Self]: """Get the fixes that must be applied for a given dataset. It will look for them at the module @@ -248,15 +250,25 @@ def get_fixes( Returns ------- - list[Fix] + : Fixes to apply for the given data. """ + if session is None: + warnings.warn( + "Not providing a `session` argument or using `session=None` " + "is deprecated and will no longer be supported in v2.16.0.", + DeprecationWarning, + stacklevel=2, + ) + from esmvalcore.config import CFG # noqa: PLC0415 + + session = CFG.start_session("fix") + if extra_facets is None: extra_facets = {} - vardef = get_var_info( - project, + vardef = get_tables(session, project).get_variable( mip, short_name, branding_suffix=extra_facets.get("branding_suffix"), @@ -271,7 +283,6 @@ def get_fixes( fixes_modules = [] if project == "cordex": driver = extra_facets["driver"].replace("-", "_").lower() - extra_facets["dataset"] = dataset with contextlib.suppress(ImportError): fixes_modules.append( importlib.import_module( @@ -374,7 +385,10 @@ def fix_metadata(self, cubes: Sequence[Cube]) -> CubeList: Fixed cubes. """ - # Make sure the this fix also works when no extra_facets are given + # Make sure the this fix also works when no extra_facets are given. + # Note that this never happens in practice because `"project"` and + # `"dataset"` are inserted into `extra_facets` in + # `esmvalcore.cmor.fix.fix_*`. if "project" in self.extra_facets and "dataset" in self.extra_facets: dataset_str = ( f"{self.extra_facets['project']}:" @@ -600,7 +614,8 @@ def _fix_alternative_generic_level_coords(self, cube: Cube) -> Cube: _get_alternative_generic_lev_coord( cube, coord_name, - self.vardef.table_type, + project=self.extra_facets["project"], + session=self.session, ) ) except ValueError: # no alternatives found diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py index f939caadf3..dae9a871d6 100644 --- a/esmvalcore/cmor/_utils.py +++ b/esmvalcore/cmor/_utils.py @@ -5,7 +5,7 @@ import logging from typing import TYPE_CHECKING -from esmvalcore.cmor.table import CMOR_TABLES +from esmvalcore.cmor.table import get_tables if TYPE_CHECKING: from collections.abc import Sequence @@ -14,17 +14,18 @@ from iris.cube import Cube from esmvalcore.cmor.table import CoordinateInfo, VariableInfo + from esmvalcore.config import Session logger = logging.getLogger(__name__) _ALTERNATIVE_GENERIC_LEV_COORDS = { "alevel": { - "CMIP5": ["alt40", "plevs"], - "CMIP6": ["alt16", "plev3"], - "obs4MIPs": ["alt16", "plev3"], + "CMIP5Info": ["alt40", "plevs"], + "CMIP6Info": ["alt16", "plev3"], + "Obs4MIPsInfo": ["alt16", "plev3"], }, "zlevel": { - "CMIP3": ["pressure"], + "CMIP3Info": ["pressure"], }, } @@ -32,7 +33,8 @@ def _get_alternative_generic_lev_coord( cube: Cube, coord_name: str, - cmor_table_type: str, + project: str, + session: Session, ) -> tuple[CoordinateInfo, Coord]: """Find alternative generic level coordinate in cube. @@ -42,10 +44,10 @@ def _get_alternative_generic_lev_coord( Cube to be checked. coord_name: Name of the generic level coordinate. - cmor_table_type: - CMOR table type, e.g., CMIP3, CMIP5, CMIP6. Note: This is NOT the - project of the dataset, but rather the entry `cmor_type` in - `config-developer.yml`. + project: + Project that the dataset belongs to. + session: + The session to use. Returns ------- @@ -63,12 +65,16 @@ def _get_alternative_generic_lev_coord( coord_name, {}, ) - allowed_alternatives = alternatives_for_coord.get(cmor_table_type, []) + tables = get_tables(session, project) + allowed_alternatives = alternatives_for_coord.get( + tables.__class__.__name__, + [], + ) # Check if any of the allowed alternative coordinates is present in the # cube for allowed_alternative in allowed_alternatives: - cmor_coord = CMOR_TABLES[cmor_table_type].coords[allowed_alternative] + cmor_coord = tables.coords[allowed_alternative] if cube.coords(var_name=cmor_coord.out_name): cube_coord = cube.coord(var_name=cmor_coord.out_name) return (cmor_coord, cube_coord) @@ -77,9 +83,7 @@ def _get_alternative_generic_lev_coord( f"Found no valid alternative coordinate for generic level coordinate " f"'{coord_name}'" ) - raise ValueError( - msg, - ) + raise ValueError(msg) def _get_generic_lev_coord_names( diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 3e27da363d..d01ef483f0 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import warnings from enum import IntEnum from functools import cached_property from typing import TYPE_CHECKING, NamedTuple @@ -22,7 +23,7 @@ _get_new_generic_level_coord, _get_simplified_calendar, ) -from esmvalcore.cmor.table import get_var_info +from esmvalcore.cmor.table import get_tables from esmvalcore.iris_helpers import has_unstructured_grid if TYPE_CHECKING: @@ -32,6 +33,7 @@ from iris.cube import Cube from esmvalcore.cmor.table import CoordinateInfo + from esmvalcore.config import Session class CheckLevels(IntEnum): @@ -938,14 +940,24 @@ def _get_cmor_checker( mip: str, short_name: str, *, + session: Session | None, branding_suffix: str | None = None, frequency: None | str = None, - fail_on_error: bool = False, check_level: CheckLevels = CheckLevels.DEFAULT, ) -> Callable[[Cube], CMORCheck]: """Get a CMOR checker.""" - var_info = get_var_info( - project, + if session is None: + warnings.warn( + "Not providing a `session` argument or using `session=None` " + "is deprecated and will no longer be supported in v2.16.0.", + DeprecationWarning, + stacklevel=2, + ) + from esmvalcore.config import CFG # noqa: PLC0415 + + session = CFG.start_session("cmor_check") + + var_info = get_tables(session, project).get_variable( mip, short_name, branding_suffix=branding_suffix, @@ -956,7 +968,6 @@ def _checker(cube: Cube) -> CMORCheck: cube, var_info, frequency=frequency, - fail_on_error=fail_on_error, check_level=check_level, ) @@ -969,6 +980,7 @@ def cmor_check_metadata( mip: str, short_name: str, *, + session: Session | None = None, branding_suffix: str | None = None, frequency: str | None = None, check_level: CheckLevels = CheckLevels.DEFAULT, @@ -987,6 +999,8 @@ def cmor_check_metadata( Variable's MIP. short_name: Variable's short name. + session: + The session to use. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the variable in the CMOR table. Used by the CMIP7 project. @@ -996,6 +1010,11 @@ def cmor_check_metadata( check_level: Level of strictness of the checks. + .. deprecated: 2.14.0 + The ``check_level`` parameter is deprecated and will be removed in + version 2.16.0. Please set the desired strictness level using + ``session["check_level"]`` instead. + Returns ------- iris.cube.Cube @@ -1008,6 +1027,7 @@ def cmor_check_metadata( short_name, branding_suffix=branding_suffix, frequency=frequency, + session=session, check_level=check_level, ) return checker(cube).check_metadata() @@ -1019,6 +1039,7 @@ def cmor_check_data( mip: str, short_name: str, *, + session: Session | None = None, branding_suffix: str | None = None, frequency: str | None = None, check_level: CheckLevels = CheckLevels.DEFAULT, @@ -1035,6 +1056,8 @@ def cmor_check_data( Variable's MIP. short_name: Variable's short name + session: + The session to use. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the variable in the CMOR table. Used by the CMIP7 project. @@ -1044,6 +1067,11 @@ def cmor_check_data( check_level: Level of strictness of the checks. + .. deprecated: 2.14.0 + The ``check_level`` parameter is deprecated and will be removed in + version 2.16.0. Please set the desired strictness level using + ``session["check_level"]`` instead. + Returns ------- iris.cube.Cube @@ -1056,6 +1084,7 @@ def cmor_check_data( short_name, branding_suffix=branding_suffix, frequency=frequency, + session=session, check_level=check_level, ) return checker(cube).check_data() @@ -1067,6 +1096,7 @@ def cmor_check( mip: str, short_name: str, *, + session: Session | None = None, branding_suffix: str | None = None, frequency: str | None = None, check_level: CheckLevels = CheckLevels.DEFAULT, @@ -1086,6 +1116,8 @@ def cmor_check( Variable's MIP. short_name: Variable's short name. + session: + The session to use. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the variable in the CMOR table. Used by the CMIP7 project. @@ -1095,6 +1127,11 @@ def cmor_check( check_level: Level of strictness of the checks. + .. deprecated: 2.14.0 + The ``check_level`` parameter is deprecated and will be removed in + version 2.16.0. Please set the desired strictness level using + ``session["check_level"]`` instead. + Returns ------- iris.cube.Cube @@ -1108,6 +1145,7 @@ def cmor_check( short_name, branding_suffix=branding_suffix, frequency=frequency, + session=session, check_level=check_level, ) return cmor_check_data( @@ -1117,5 +1155,6 @@ def cmor_check( short_name, branding_suffix=branding_suffix, frequency=frequency, + session=session, check_level=check_level, ) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 7a083ff58d..ae310a9449 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -31,7 +31,14 @@ logger = logging.getLogger(__name__) CMOR_TABLES: dict[str, InfoBase] = {} -"""dict of str, obj: CMOR info objects.""" +"""dict of str, obj: CMOR info objects. + +.. deprecated:: 2.14.0 + + The global ``CMOR_TABLES`` dictionary is deprecated and will be removed in + ESMValCore v2.16.0. Please use :func:`~esmvalcore.cmor.table.get_tables` + to access the CMOR tables instead. +""" _CMOR_KEYS = ( "standard_name", @@ -140,6 +147,12 @@ def get_var_info( ) -> VariableInfo | None: """Get variable information. + .. deprecated:: 2.14.0 + The ``get_var_info`` function is deprecated and will be removed in + ESMValCore v2.16.0. Please use :func:`~esmvalcore.cmor.table.get_tables` + to retrieve the tables for a project and :meth:`InfoBase.get_variable` + to retrieve a variable from a table. + Note ---- If `project=CORDEX` and the `mip` ends with 'hr', it is cropped to 'h' @@ -188,21 +201,6 @@ def get_var_info( ) -def load_cmor_tables(cfg: Config) -> None: - """Load the configured CMOR tables into :data:`esmvalcore.cmor.table.CMOR_TABLES`. - - Parameters - ---------- - cfg: - The configuration. - """ - CMOR_TABLES.clear() - if cfg.get("config_developer_file") is not None: - read_cmor_tables(cfg["config_developer_file"]) - for project in cfg["projects"]: - CMOR_TABLES[project] = get_tables(cfg, project) - - def read_cmor_tables(cfg_developer: Path | None = None) -> None: """Read cmor tables required in the configuration. @@ -210,7 +208,7 @@ def read_cmor_tables(cfg_developer: Path | None = None) -> None: The config-developer.yml file based configuration is deprecated and will no longer be supported in ESMValCore v2.16.0. Please use - :func:`~esmvalcore.cmor.table.load_cmor_tables` instead of this function. + :func:`~esmvalcore.cmor.table.get_tables` instead of this function. Parameters ---------- @@ -457,6 +455,9 @@ def __init__( self.tables: dict[str, TableInfo] = {} """A mapping from table names to :class:`TableInfo` objects.""" + def __repr__(self) -> str: + return f"{self.__class__.__name__}(paths={self.paths}, strict={self.strict})" + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. @@ -934,6 +935,8 @@ def __init__( table_type: str = "", short_name: str = "", name: str = "", + table: str = "", + project: str = "", ) -> None: """Class to read and store variable information. @@ -953,12 +956,21 @@ def __init__( The ``short_name`` parameter is deprecated and will be removed in ESMValCore v2.16.0. + # TODO: maybe not have these? They come from an upper level. name: Name of the variable entry in the CMOR table. + table: + Name of the table where the variable is defined. + project: + Name of the project the variable belongs to. """ super().__init__() self.name = name """Name of the variable entry in the CMOR table.""" + self.table = table + """Name of the table where the variable is defined.""" + self.project = project + """Name of the project the variable belongs to.""" self.table_type = table_type self.modeling_realm: list[str] = [] """Modeling realm""" @@ -987,6 +999,13 @@ def __init__( This is a dict with the names of the dimensions as keys and CoordinateInfo objects as values. """ + self._alternative_coordinates: dict[str, list[str]] = {} + """Alternative coordinates that can be used for variants of this variable. + + This is a dict with the names of the dimensions as keys and + lists of alternative coordinate names as values and can be used + for variants of this variable that are similar enough for comparison. + """ self._json_data = None diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index 8a6a20c156..934392d9b7 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -874,7 +874,6 @@ def _load(self) -> Cube: } settings["concatenate"] = {"check_level": self.session["check_level"]} settings["cmor_check_metadata"] = { - "check_level": self.session["check_level"], "cmor_table": self.facets["project"], "mip": self.facets["mip"], "frequency": self.facets["frequency"], @@ -890,7 +889,6 @@ def _load(self) -> Cube: **self.facets, } settings["cmor_check_data"] = { - "check_level": self.session["check_level"], "cmor_table": self.facets["project"], "mip": self.facets["mip"], "frequency": self.facets["frequency"], diff --git a/esmvalcore/preprocessor/_other.py b/esmvalcore/preprocessor/_other.py index 972f93acbb..4309a4ce7c 100644 --- a/esmvalcore/preprocessor/_other.py +++ b/esmvalcore/preprocessor/_other.py @@ -14,7 +14,7 @@ from iris.cube import Cube from iris.exceptions import CoordinateMultiDimError -from esmvalcore.cmor.table import get_var_info +from esmvalcore.cmor.table import get_tables from esmvalcore.iris_helpers import ( ignore_iris_vague_metadata_warnings, rechunk_cube, @@ -34,6 +34,7 @@ from iris.coords import Coord from esmvalcore.cmor.table import VariableInfo + from esmvalcore.config import Session logger = logging.getLogger(__name__) @@ -101,13 +102,28 @@ def align_metadata( return cube -def _get_var_info(project: str, mip: str, short_name: str) -> VariableInfo: +def _get_var_info( + project: str, + mip: str, + short_name: str, + branding_suffix: str | None, + session: Session, +) -> VariableInfo: """Get variable information.""" - var_info = get_var_info(project, mip, short_name) + var_info = get_tables(session, project).get_variable( + table_name=mip, + short_name=short_name, + branding_suffix=branding_suffix, + ) if var_info is None: msg = ( - f"Variable '{short_name}' not available for table '{mip}' of " - f"project '{project}'" + f"Variable '{short_name}' " + + ( + f"with branding suffix '{branding_suffix}' " + if branding_suffix + else "" + ) + + f"not available for table '{mip}' of project '{project}'" ) raise ValueError(msg) return var_info diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 486b831e79..f7e2dcee21 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -32,7 +32,7 @@ add_altitude_from_plev, add_plev_from_altitude, ) -from esmvalcore.cmor.table import CMOR_TABLES +from esmvalcore.cmor.table import get_tables from esmvalcore.iris_helpers import has_irregular_grid, has_unstructured_grid from esmvalcore.preprocessor._shared import ( _rechunk_aux_factory_dependencies, @@ -57,6 +57,7 @@ from iris.analysis import Regridder, RegriddingScheme from numpy.typing import ArrayLike + from esmvalcore.config import Session from esmvalcore.dataset import Dataset logger = logging.getLogger(__name__) @@ -1383,7 +1384,11 @@ def extract_levels( return result -def get_cmor_levels(cmor_table: str, coordinate: str) -> list[float]: +def get_cmor_levels( + cmor_table: str, + coordinate: str, + session: Session, +) -> list[float]: """Get level definition from a CMOR coordinate. Parameters @@ -1392,10 +1397,13 @@ def get_cmor_levels(cmor_table: str, coordinate: str) -> list[float]: CMOR table name coordinate: CMOR coordinate name + session: + The session to use. Returns ------- - list[float] + : + A list of levels. Raises ------ @@ -1403,15 +1411,13 @@ def get_cmor_levels(cmor_table: str, coordinate: str) -> list[float]: If the CMOR table is not defined, the coordinate does not specify any levels or the string is badly formatted. """ - if cmor_table not in CMOR_TABLES: - msg = f"Level definition cmor_table '{cmor_table}' not available" - raise ValueError(msg) + cmor_tables = get_tables(session, project=cmor_table) - if coordinate not in CMOR_TABLES[cmor_table].coords: + if coordinate not in cmor_tables.coords: msg = f"Coordinate {coordinate} not available for {cmor_table}" raise ValueError(msg) - cmor = CMOR_TABLES[cmor_table].coords[coordinate] + cmor = cmor_tables.coords[coordinate] if cmor.requested: return [float(level) for level in cmor.requested]