CFE-4708: Added support for a path-variable in def.json to specify files to ship beyond just extension-matching

cfbs/build.py

@@ -13,6 +13,7 @@
 import json
 import os
 import logging as log
+import re
 import shutil
 import subprocess
 from cfbs.augments import generate_augment
@@ -286,19 +287,27 @@ def _localize_file_inputs(name, input_data, destination, build_modules):
     If a response is already shipped by another module's own "directory"
     build step (e.g. the project author set one up manually), that step's
     destination is used instead of making a redundant copy.
+
+    Returns the masterfiles-relative destination path of every file that was
+    localized, so callers can make sure those exact paths get synced by the
+    policy update mechanism even if their extension isn't one of the ones
+    normally recognized (see `input_paths_extra`).
     """
     if not isinstance(input_data, list):
-        return
+        return []
 
     module_dir_name = name[2:] if name.startswith("./") else name
     module_dir_name = os.path.basename(module_dir_name.rstrip("/"))
 
+    localized_paths = []
+
     def _localize(rel_path):
         if not rel_path or not os.path.isfile(rel_path):
             return rel_path
 
         already_shipped = _path_if_already_shipped(rel_path, build_modules, destination)
         if already_shipped is not None:
+            localized_paths.append(strip_left(already_shipped, "$(sys.inputdir)/"))
             return already_shipped
 
         dest = os.path.join(
@@ -309,7 +318,9 @@ def _localize(rel_path):
             rel_path,
         )
         cp(rel_path, dest)
-        return "$(sys.inputdir)/" + os.path.relpath(dest, destination)
+        dest_rel = os.path.relpath(dest, destination)
+        localized_paths.append(dest_rel)
+        return "$(sys.inputdir)/" + dest_rel
 
     for element in input_data:
         if not isinstance(element, dict) or element.get("type") != "file":
@@ -320,6 +331,8 @@ def _localize(rel_path):
         else:
             element["response"] = _localize(response)
 
+    return localized_paths
+
 
 def _perform_input_step(args, name, destination, prefix, build_modules):
     src, dst = args
@@ -342,14 +355,26 @@ def _perform_input_step(args, name, destination, prefix, build_modules):
         )
         return
     extras, original = read_json(src), read_json(dst)
-    _localize_file_inputs(name, extras, destination, build_modules)
+    localized_paths = _localize_file_inputs(name, extras, destination, build_modules)
     extras = generate_augment(name, extras)
     log.debug("Generated augment: %s", pretty(extras))
     if not extras:
         raise CFBSExitError(
             "Input data '%s' is incomplete: Skipping build step."
             % os.path.basename(src)
         )
+    if localized_paths:
+        # Files brought in through "file" type inputs aren't necessarily
+        # matched by the policy update's default `input_name_patterns`
+        # Rather than widening that extension-based matching for the whole policy set,
+        # point at exactly these files.
+        patterns = [
+            "$(sys.inputdir)/%s$" % re.escape(path.replace(os.sep, "/"))
+            for path in localized_paths
+        ]
+        extras = merge_json(
+            extras, {"vars": {"default:update_def.input_paths_extra": patterns}}
+        )
     if original:
         log.debug("Original def.json: %s", pretty(original))
         merged = merge_json(original, extras)

tests/test_build.py

@@ -1,7 +1,9 @@
 import os
 import copy
+import json
+import re
 
-from cfbs.build import _localize_file_inputs
+from cfbs.build import _localize_file_inputs, _perform_input_step
 
 
 def test_localize_file_inputs_copies_single_file(tmp_path, monkeypatch):
@@ -18,11 +20,14 @@ def test_localize_file_inputs_copies_single_file(tmp_path, monkeypatch):
         }
     ]
 
-    _localize_file_inputs("run-a-script", input_data, "out/masterfiles", [])
+    localized_paths = _localize_file_inputs(
+        "run-a-script", input_data, "out/masterfiles", []
+    )
 
     expected_dest = "out/masterfiles/services/cfbs/deploy.sh"
     assert os.path.isfile(expected_dest)
     assert input_data[0]["response"] == "$(sys.inputdir)/services/cfbs/deploy.sh"
+    assert localized_paths == ["services/cfbs/deploy.sh"]
 
 
 def test_localize_file_inputs_copies_list_of_files(tmp_path, monkeypatch):
@@ -42,7 +47,9 @@ def test_localize_file_inputs_copies_list_of_files(tmp_path, monkeypatch):
         }
     ]
 
-    _localize_file_inputs("run-scripts", input_data, "out/masterfiles", [])
+    localized_paths = _localize_file_inputs(
+        "run-scripts", input_data, "out/masterfiles", []
+    )
 
     assert input_data[0]["response"] == [
         "$(sys.inputdir)/services/cfbs/one.sh",
@@ -52,6 +59,10 @@ def test_localize_file_inputs_copies_list_of_files(tmp_path, monkeypatch):
     assert os.path.isfile(
         "out/masterfiles/services/cfbs/modules/run-scripts-module/two.sh"
     )
+    assert localized_paths == [
+        "services/cfbs/one.sh",
+        "services/cfbs/modules/run-scripts-module/two.sh",
+    ]
 
 
 def test_localize_file_inputs_strips_local_module_prefix(tmp_path, monkeypatch):
@@ -164,3 +175,54 @@ def test_localize_file_inputs_ignores_non_directory_steps(tmp_path, monkeypatch)
         "$(sys.inputdir)/services/cfbs/modules/run-scripts/deploy.sh"
     )
     assert os.path.isfile("out/masterfiles/services/cfbs/modules/run-scripts/deploy.sh")
+
+
+def test_perform_input_step_adds_input_paths_extra_for_localized_files(
+    tmp_path, monkeypatch
+):
+    """A "file" type input should make the build add its exact destination
+    path to `default:update_def.input_paths_extra`, so the policy update
+    mechanism syncs it even if its extension isn't in the default
+    `input_name_patterns` list.
+    """
+    monkeypatch.chdir(tmp_path)
+    os.makedirs("out/masterfiles")
+    os.makedirs("run-a-script")
+    with open("deploy.sh", "w") as f:
+        f.write("echo hi\n")
+    with open("run-a-script/input.json", "w") as f:
+        json.dump([{"type": "file", "variable": "script", "response": "deploy.sh"}], f)
+
+    _perform_input_step(
+        ["./input.json", "def.json"], "run-a-script", "out/masterfiles", "+", []
+    )
+
+    with open("out/masterfiles/def.json") as f:
+        result = json.load(f)
+
+    expected_path = "services/cfbs/deploy.sh"
+    assert result["vars"]["default:update_def.input_paths_extra"] == [
+        "$(sys.inputdir)/%s$" % re.escape(expected_path)
+    ]
+
+
+def test_perform_input_step_skips_input_paths_extra_for_non_file_inputs(
+    tmp_path, monkeypatch
+):
+    """A build with only "string"-type inputs has nothing to localize, so no
+    `input_paths_extra` augment should be generated at all.
+    """
+    monkeypatch.chdir(tmp_path)
+    os.makedirs("out/masterfiles")
+    os.makedirs("some-module")
+    with open("some-module/input.json", "w") as f:
+        json.dump([{"type": "string", "variable": "greeting", "response": "hello"}], f)
+
+    _perform_input_step(
+        ["./input.json", "def.json"], "some-module", "out/masterfiles", "+", []
+    )
+
+    with open("out/masterfiles/def.json") as f:
+        result = json.load(f)
+
+    assert "vars" not in result