diff --git a/cfbs/build.py b/cfbs/build.py index 86af077c..e4e76a58 100644 --- a/cfbs/build.py +++ b/cfbs/build.py @@ -13,6 +13,7 @@ import json import os import logging as log +import re import shutil import subprocess from cfbs.augments import generate_augment @@ -286,19 +287,27 @@ def _localize_file_inputs(name, input_data, destination, build_modules): If a response is already shipped by another module's own "directory" build step (e.g. the project author set one up manually), that step's destination is used instead of making a redundant copy. + + Returns the masterfiles-relative destination path of every file that was + localized, so callers can make sure those exact paths get synced by the + policy update mechanism even if their extension isn't one of the ones + normally recognized (see `input_paths_extra`). """ if not isinstance(input_data, list): - return + return [] module_dir_name = name[2:] if name.startswith("./") else name module_dir_name = os.path.basename(module_dir_name.rstrip("/")) + localized_paths = [] + def _localize(rel_path): if not rel_path or not os.path.isfile(rel_path): return rel_path already_shipped = _path_if_already_shipped(rel_path, build_modules, destination) if already_shipped is not None: + localized_paths.append(strip_left(already_shipped, "$(sys.inputdir)/")) return already_shipped dest = os.path.join( @@ -309,7 +318,9 @@ def _localize(rel_path): rel_path, ) cp(rel_path, dest) - return "$(sys.inputdir)/" + os.path.relpath(dest, destination) + dest_rel = os.path.relpath(dest, destination) + localized_paths.append(dest_rel) + return "$(sys.inputdir)/" + dest_rel for element in input_data: if not isinstance(element, dict) or element.get("type") != "file": @@ -320,6 +331,8 @@ def _localize(rel_path): else: element["response"] = _localize(response) + return localized_paths + def _perform_input_step(args, name, destination, prefix, build_modules): src, dst = args @@ -342,7 +355,7 @@ def _perform_input_step(args, name, destination, prefix, build_modules): ) return extras, original = read_json(src), read_json(dst) - _localize_file_inputs(name, extras, destination, build_modules) + localized_paths = _localize_file_inputs(name, extras, destination, build_modules) extras = generate_augment(name, extras) log.debug("Generated augment: %s", pretty(extras)) if not extras: @@ -350,6 +363,18 @@ def _perform_input_step(args, name, destination, prefix, build_modules): "Input data '%s' is incomplete: Skipping build step." % os.path.basename(src) ) + if localized_paths: + # Files brought in through "file" type inputs aren't necessarily + # matched by the policy update's default `input_name_patterns` + # Rather than widening that extension-based matching for the whole policy set, + # point at exactly these files. + patterns = [ + "$(sys.inputdir)/%s$" % re.escape(path.replace(os.sep, "/")) + for path in localized_paths + ] + extras = merge_json( + extras, {"vars": {"default:update_def.input_paths_extra": patterns}} + ) if original: log.debug("Original def.json: %s", pretty(original)) merged = merge_json(original, extras) diff --git a/tests/test_build.py b/tests/test_build.py index 7400388c..8d278ae7 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1,7 +1,9 @@ import os import copy +import json +import re -from cfbs.build import _localize_file_inputs +from cfbs.build import _localize_file_inputs, _perform_input_step def test_localize_file_inputs_copies_single_file(tmp_path, monkeypatch): @@ -18,11 +20,14 @@ def test_localize_file_inputs_copies_single_file(tmp_path, monkeypatch): } ] - _localize_file_inputs("run-a-script", input_data, "out/masterfiles", []) + localized_paths = _localize_file_inputs( + "run-a-script", input_data, "out/masterfiles", [] + ) expected_dest = "out/masterfiles/services/cfbs/deploy.sh" assert os.path.isfile(expected_dest) assert input_data[0]["response"] == "$(sys.inputdir)/services/cfbs/deploy.sh" + assert localized_paths == ["services/cfbs/deploy.sh"] def test_localize_file_inputs_copies_list_of_files(tmp_path, monkeypatch): @@ -42,7 +47,9 @@ def test_localize_file_inputs_copies_list_of_files(tmp_path, monkeypatch): } ] - _localize_file_inputs("run-scripts", input_data, "out/masterfiles", []) + localized_paths = _localize_file_inputs( + "run-scripts", input_data, "out/masterfiles", [] + ) assert input_data[0]["response"] == [ "$(sys.inputdir)/services/cfbs/one.sh", @@ -52,6 +59,10 @@ def test_localize_file_inputs_copies_list_of_files(tmp_path, monkeypatch): assert os.path.isfile( "out/masterfiles/services/cfbs/modules/run-scripts-module/two.sh" ) + assert localized_paths == [ + "services/cfbs/one.sh", + "services/cfbs/modules/run-scripts-module/two.sh", + ] def test_localize_file_inputs_strips_local_module_prefix(tmp_path, monkeypatch): @@ -164,3 +175,54 @@ def test_localize_file_inputs_ignores_non_directory_steps(tmp_path, monkeypatch) "$(sys.inputdir)/services/cfbs/modules/run-scripts/deploy.sh" ) assert os.path.isfile("out/masterfiles/services/cfbs/modules/run-scripts/deploy.sh") + + +def test_perform_input_step_adds_input_paths_extra_for_localized_files( + tmp_path, monkeypatch +): + """A "file" type input should make the build add its exact destination + path to `default:update_def.input_paths_extra`, so the policy update + mechanism syncs it even if its extension isn't in the default + `input_name_patterns` list. + """ + monkeypatch.chdir(tmp_path) + os.makedirs("out/masterfiles") + os.makedirs("run-a-script") + with open("deploy.sh", "w") as f: + f.write("echo hi\n") + with open("run-a-script/input.json", "w") as f: + json.dump([{"type": "file", "variable": "script", "response": "deploy.sh"}], f) + + _perform_input_step( + ["./input.json", "def.json"], "run-a-script", "out/masterfiles", "+", [] + ) + + with open("out/masterfiles/def.json") as f: + result = json.load(f) + + expected_path = "services/cfbs/deploy.sh" + assert result["vars"]["default:update_def.input_paths_extra"] == [ + "$(sys.inputdir)/%s$" % re.escape(expected_path) + ] + + +def test_perform_input_step_skips_input_paths_extra_for_non_file_inputs( + tmp_path, monkeypatch +): + """A build with only "string"-type inputs has nothing to localize, so no + `input_paths_extra` augment should be generated at all. + """ + monkeypatch.chdir(tmp_path) + os.makedirs("out/masterfiles") + os.makedirs("some-module") + with open("some-module/input.json", "w") as f: + json.dump([{"type": "string", "variable": "greeting", "response": "hello"}], f) + + _perform_input_step( + ["./input.json", "def.json"], "some-module", "out/masterfiles", "+", [] + ) + + with open("out/masterfiles/def.json") as f: + result = json.load(f) + + assert "vars" not in result