Merge 5 bit-for-bit pull requests:

* Add system and unit tests for making fsurdat with all crops everywhere (#2081)
* Rework master_list* files etc. (#2087)
* Fixes to methane Tech Note (#2091)
* Add is_doy_in_interval() function (#2158)
* Avoid using subprocess.run() in FSURDATMODIFYCTSM (#2125)

Closes issues:
* Add unit test for making fsurdat with all crops everywhere (#2079)
* Rework master_list_(no)?fates.rst? (#2083)
* conda run -n can fail if a conda environment is already active (#2109)
* conda fails to load for SystemTests (#2111)

Author: samsrabin

.git-blame-ignore-revs

@@ -1,4 +1,5 @@
 # Ran python directory through black python formatter
+d229b5c6689efc4c2a6cef077515c4ccd5c18ff6
 4cd83cb3ee6d85eb909403487abf5eeaf4d98911
 0aa2957c1f8603c63fa30b11295c06cfddff44a5
 2cdb380febb274478e84cd90945aee93f29fa2e6
@@ -10,6 +11,11 @@ b88e1cd1b28e3609684c79a2ec0e88f26cfc362b
 b771971e3299c4fa56534b93421f7a2b9c7282fd
 9de88bb57ea9855da408cbec1dc8acb9079eda47
 8bc4688e52ea23ef688e283698f70a44388373eb
+0a5a9e803b56ec1bbd6232eff1c99dbbeef25eb7
+810cb346f05ac1aabfff931ab1a2b7b584add241
+5933b0018f8e29413e30dda9b906370d147bad45
 # Ran SystemTests and python/ctsm through black python formatter
 5364ad66eaceb55dde2d3d598fe4ce37ac83a93c
 8056ae649c1b37f5e10aaaac79005d6e3a8b2380
+540b256d1f3382f4619d7b0877c32d54ce5c40b6
+8a168bb0895f4f2421608dd2589398e13a6663e6

bld/CLMBuildNamelist.pm

@@ -4413,7 +4413,7 @@ sub setup_logic_misc {
    add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'for_testing_use_second_grain_pool');
    add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'for_testing_use_repr_structure_pool');
    add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'for_testing_no_crop_seed_replenishment');
-   add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'hist_master_list_file');
+   add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'hist_fields_list_file');
 }
 
 #-------------------------------------------------------------------------------

bld/namelist_files/namelist_defaults_ctsm.xml

@@ -67,8 +67,8 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
 <for_testing_use_repr_structure_pool>.false.</for_testing_use_repr_structure_pool>
 <for_testing_no_crop_seed_replenishment>.false.</for_testing_no_crop_seed_replenishment>
 
-<!-- Set to .true. in namelist to write hist fields master list file -->
-<hist_master_list_file>.false.</hist_master_list_file>
+<!-- Set to .true. in namelist to write file with all history fields -->
+<hist_fields_list_file>.false.</hist_fields_list_file>
 
 <!-- In accelerated spinup mode reduce the amount of history output -->
 <hist_empty_htapes clm_accelerated_spinup="on">.true.</hist_empty_htapes>

bld/namelist_files/namelist_definition_ctsm.xml

@@ -764,9 +764,9 @@ SNICAR (SNow, ICe, and Aerosol Radiative model) optical data file name
 SNICAR (SNow, ICe, and Aerosol Radiative model) snow aging data file name
 </entry>
 
-<entry id="hist_master_list_file" type="logical" category="history"
+<entry id="hist_fields_list_file" type="logical" category="history"
        group="clm_inparm" valid_values="" value=".false.">
-If TRUE, write master field list to separate file for documentation purposes
+If TRUE, write list of all output fields to separate file for documentation purposes
 </entry>
 
 <entry id="hist_avgflag_pertape" type="char*10(10)" category="history"

cime_config/SystemTests/fsurdatmodifyctsm.py

@@ -5,11 +5,18 @@
 
 import os
 import re
-import systemtest_utils as stu
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.test_utils.user_nl_utils import append_to_user_nl_files
 
+# For calling fsurdat_modifier
+from argparse import Namespace
+
+_CTSM_PYTHON = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python"
+)
+sys.path.insert(1, _CTSM_PYTHON)
+
 logger = logging.getLogger(__name__)
 
 
@@ -66,16 +73,18 @@ def _create_config_file(self):
                     cfg_out.write(line)
 
     def _run_modify_fsurdat(self):
-        tool_path = os.path.join(self._ctsm_root, "tools/modify_input_files/fsurdat_modifier")
-
-        self._case.load_env(reset=True)
-        command = f"python3 {tool_path} {self._cfg_file_path}"
-        stu.run_python_script(
-            self._get_caseroot(),
-            "ctsm_pylib",
-            command,
-            tool_path,
+        fsurdat_modifier_args = Namespace(
+            cfg_path=self._cfg_file_path,
+            debug=False,
+            fsurdat_in="UNSET",
+            fsurdat_out="UNSET",
+            overwrite=False,
+            silent=False,
+            verbose=False,
         )
+        from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier
+
+        fsurdat_modifier(fsurdat_modifier_args)
 
     def _modify_user_nl(self):
         append_to_user_nl_files(

cime_config/SystemTests/rxcropmaturity.py

@@ -134,8 +134,8 @@ def run_phase(self):
             case_gddgen.check_all_input_data()
 
             # Make custom version of surface file
-            logger.info("RXCROPMATURITY log:  run make_fsurdat_all_crops_everywhere")
-            self._run_make_fsurdat_all_crops_everywhere()
+            logger.info("RXCROPMATURITY log:  run fsurdat_modifier")
+            self._run_fsurdat_modifier()
 
         # -------------------------------------------------------------------
         # (2) Perform GDD-generating run and generate prescribed GDDs file
@@ -239,7 +239,7 @@ def _setup_all(self):
         logger.info("RXCROPMATURITY log:  _setup_all done")
 
     # Make a surface dataset that has every crop in every gridcell
-    def _run_make_fsurdat_all_crops_everywhere(self):
+    def _run_fsurdat_modifier(self):
 
         # fsurdat should be defined. Where is it?
         self._fsurdat_in = None
@@ -255,20 +255,29 @@ def _run_make_fsurdat_all_crops_everywhere(self):
             raise RuntimeError(error_message)
 
         # Where we will save the fsurdat version for this test
-        self._fsurdat_out = os.path.join(self._path_gddgen, "fsurdat.nc")
+        path, ext = os.path.splitext(self._fsurdat_in)
+        dir_in, filename_in_noext = os.path.split(path)
+        self._fsurdat_out = os.path.join(
+            self._path_gddgen, f"{filename_in_noext}.all_crops_everywhere{ext}"
+        )
 
         # Make fsurdat for this test, if not already done
         if not os.path.exists(self._fsurdat_out):
             tool_path = os.path.join(
                 self._ctsm_root,
-                "python",
-                "ctsm",
-                "crop_calendars",
-                "make_fsurdat_all_crops_everywhere.py",
+                "tools",
+                "modify_input_files",
+                "fsurdat_modifier",
             )
-            command = (
-                f"python3 {tool_path} " + f"-i {self._fsurdat_in} " + f"-o {self._fsurdat_out}"
+
+            # Create configuration file for fsurdat_modifier
+            self._cfg_path = os.path.join(
+                self._path_gddgen,
+                "modify_fsurdat_allcropseverywhere.cfg",
             )
+            self._create_config_file_evenlysplitcrop()
+
+            command = f"python3 {tool_path} {self._cfg_path} "
             stu.run_python_script(
                 self._get_caseroot(),
                 self._this_conda_env,
@@ -287,6 +296,40 @@ def _run_make_fsurdat_all_crops_everywhere(self):
             ]
         )
 
+    def _create_config_file_evenlysplitcrop(self):
+        """
+        Open the new and the template .cfg files
+        Loop line by line through the template .cfg file
+        When string matches, replace that line's content
+        """
+        cfg_template_path = os.path.join(
+            self._ctsm_root, "tools/modify_input_files/modify_fsurdat_template.cfg"
+        )
+
+        with open(self._cfg_path, "w", encoding="utf-8") as cfg_out:
+            # Copy template, replacing some lines
+            with open(cfg_template_path, "r", encoding="utf-8") as cfg_in:
+                for line in cfg_in:
+                    if re.match(r" *evenly_split_cropland *=", line):
+                        line = f"evenly_split_cropland = True"
+                    elif re.match(r" *fsurdat_in *=", line):
+                        line = f"fsurdat_in = {self._fsurdat_in}"
+                    elif re.match(r" *fsurdat_out *=", line):
+                        line = f"fsurdat_out = {self._fsurdat_out}"
+                    elif re.match(r" *process_subgrid_section *=", line):
+                        line = f"process_subgrid_section = True"
+                    cfg_out.write(line)
+
+            # Add new lines
+            cfg_out.write("\n")
+            cfg_out.write("[modify_fsurdat_subgrid_fractions]\n")
+            cfg_out.write("PCT_CROP    = 100.0\n")
+            cfg_out.write("PCT_NATVEG  = 0.0\n")
+            cfg_out.write("PCT_GLACIER = 0.0\n")
+            cfg_out.write("PCT_WETLAND = 0.0\n")
+            cfg_out.write("PCT_LAKE    = 0.0\n")
+            cfg_out.write("PCT_URBAN   = 0.0 0.0 0.0\n")
+
     def _run_check_rxboth_run(self):
 
         output_dir = os.path.join(self._get_caseroot(), "run")

cime_config/SystemTests/systemtest_utils.py

@@ -10,27 +10,41 @@ def cmds_to_setup_conda(caseroot):
     # Use semicolon here since it's OK to fail
     #
     conda_setup_commands = ". " + caseroot + "/.env_mach_specific.sh; "
+    # Setting CONDA_PREFIX to empty ensures that this works even if called from
+    # a shell with a conda environment activated
+    conda_setup_commands += "CONDA_PREFIX=; "
     # Execute the module unload/load when "which conda" fails
     # eg on cheyenne
     try:
         subprocess.run("which conda", shell=True, check=True)
     except subprocess.CalledProcessError:
         # Remove python and add conda to environment for cheyennne
-        conda_setup_commands += " module unload python; module load conda;"
+        unload_python_load_conda = "module unload python; module load conda;"
+        # Make sure that adding this actually loads conda
+        subprocess.run(unload_python_load_conda + "which conda", shell=True, check=True)
+        # Save
+        conda_setup_commands += " " + unload_python_load_conda
 
     return conda_setup_commands
 
 
-def run_python_script(caseroot, this_conda_env, command, tool_path):
-
+def cmds_to_run_via_conda(caseroot, conda_run_call, command):
     # Run in the specified conda environment
     conda_setup_commands = cmds_to_setup_conda(caseroot)
-    conda_setup_commands += f" conda run -n {this_conda_env}"
+    conda_setup_commands += " " + conda_run_call
 
     # Finish with Python script call
     command = conda_setup_commands + " " + command
     print(f"command: {command}")
 
+    return command
+
+
+def run_python_script(caseroot, this_conda_env, command_in, tool_path):
+
+    # First, try with "conda run -n"
+    command = cmds_to_run_via_conda(caseroot, f"conda run -n {this_conda_env}", command_in)
+
     # Run with logfile
     tool_name = os.path.split(tool_path)[-1]
     try:
@@ -39,19 +53,34 @@ def run_python_script(caseroot, this_conda_env, command, tool_path):
                 command, shell=True, check=True, text=True, stdout=f, stderr=subprocess.STDOUT
             )
     except subprocess.CalledProcessError as error:
-        print("ERROR while getting the conda environment and/or ")
-        print(f"running the {tool_name} tool: ")
-        print(f"(1) If your {this_conda_env} environment is out of date or you ")
-        print(f"have not created the {this_conda_env} environment, yet, you may ")
-        print("get past this error by running ./py_env_create ")
-        print("in your ctsm directory and trying this test again. ")
-        print("(2) If conda is not available, install and load conda, ")
-        print("run ./py_env_create, and then try this test again. ")
-        print("(3) If (1) and (2) are not the issue, then you may be ")
-        print(f"getting an error within {tool_name} itself. ")
-        print("Default error message: ")
-        print(error.output)
-        raise
+        # Retry with the original "conda activate" method
+        command = cmds_to_run_via_conda(
+            caseroot,
+            f"conda activate {this_conda_env} && ",
+            command_in,
+        )
+        try:
+            with open(tool_name + ".log2", "w") as f:
+                subprocess.run(
+                    command, shell=True, check=True, text=True, stdout=f, stderr=subprocess.STDOUT
+                )
+        except subprocess.CalledProcessError as error:
+            print("ERROR while getting the conda environment and/or ")
+            print(f"running the {tool_name} tool: ")
+            print(f"(1) If your {this_conda_env} environment is out of date or you ")
+            print(f"have not created the {this_conda_env} environment, yet, you may ")
+            print("get past this error by running ./py_env_create ")
+            print("in your ctsm directory and trying this test again. ")
+            print("(2) If conda is not available, install and load conda, ")
+            print("run ./py_env_create, and then try this test again. ")
+            print("(3) If (1) and (2) are not the issue, then you may be ")
+            print(f"getting an error within {tool_name} itself. ")
+            print("Default error message: ")
+            print(error.output)
+            raise
+        except:
+            print(f"ERROR trying to run {tool_name}.")
+            raise
     except:
         print(f"ERROR trying to run {tool_name}.")
         raise

cime_config/testdefs/testlist_clm.xml

@@ -1007,7 +1007,7 @@
     </machines>
     <options>
       <option name="wallclock">00:20:00</option>
-      <option name="comment"  >Among other extra outputs, ensure that writing the history field master list to a separate file does not cause failure"</option>
+      <option name="comment"  >Among other extra outputs, ensure that writing the list of all history fields to a separate file does not cause failure"</option>
     </options>
   </test>
   <test name="ERS_D_Ld3" grid="f10_f10_mg37" compset="I1850Clm50BgcCrop" testmods="clm/default">
@@ -2590,7 +2590,7 @@
     </machines>
   </test>
 
-  <test name="RXCROPMATURITY_Lm61" grid="f09_g17" compset="IHistClm50BgcCrop">
+  <test name="RXCROPMATURITY_Lm61" grid="f09_g17" compset="IHistClm50BgcCrop"  testmods="clm/cropMonthOutput">
     <machines>
       <machine name="cheyenne" compiler="intel" category="ctsm_sci">
         <options>

cime_config/testdefs/testmods_dirs/clm/FatesColdCH4Off/user_nl_clm

@@ -1,2 +1,2 @@
 use_lch4 = .false.
-hist_master_list_file = .true.
+hist_fields_list_file = .true.

cime_config/testdefs/testmods_dirs/clm/SaveHistFieldList/user_nl_clm

@@ -0,0 +1 @@
+hist_fields_list_file = .true.