diff --git a/README.md b/README.md index 2f3c584..321c909 100644 --- a/README.md +++ b/README.md @@ -65,12 +65,13 @@ source ~/.bashrc **Usage**: `ddiff ` **Commands** - - `server` - Run the registry server + - `server` - Run the registry server (delete API + GC enabled) - `push` ` ... ` - Push one or more images - `pull` ` ... ` - Pull one or more images - `diff` ` ` - Diff the target image from the base image - - `load` `` - Load the target image from diff file + - `load` `() [--delete]` - Load the target image from diff file and optionally delete it from registry - `build` `` - Build the image and diff from base (FROM ...) + - `delete` `` - Delete one image tag from the registry and run garbage collection # Docker base images For easier sharing of base images, DockerDiff provides several pre-configured base images. diff --git a/ddiff.py b/ddiff.py index 0960922..2e83414 100644 --- a/ddiff.py +++ b/ddiff.py @@ -110,6 +110,27 @@ def _request_manifest(tag): print_error(f"HTTP error: {e.code} - {e.reason} ({manifest_url})") except Exception as e: print_error(f"Error: {e} ({manifest_url})") + + +def _request_manifest_digest(tag): + if "@" in tag: + repo, reference = tag.split("@", 1) + else: + repo, reference = tag.split(":", 1) + manifest_url = f"{ddiff_url}/v2/{repo}/manifests/{reference}" + + req = urllib.request.Request(manifest_url) + req.add_header("Accept", ACCEPT_MANIFEST_TYPES) + try: + with urllib.request.urlopen(req) as response: + digest = response.getheader("Docker-Content-Digest") + if not digest: + print_error(f"Could not resolve manifest digest from response headers ({manifest_url})") + return digest + except urllib.error.HTTPError as e: + print_error(f"HTTP error: {e.code} - {e.reason} ({manifest_url})") + except Exception as e: + print_error(f"Error: {e} ({manifest_url})") def _validate_manifest_media_type(manifest): media_type = manifest.get("mediaType", "") @@ -201,12 +222,40 @@ def _upload_manifest(tag, manifest_path, manifest_media_type=DOCKER_MANIFEST_V2) print_error(f"Manifest upload failed: {e.code} {e.reason} ({url})") def run_registry(): - volume_arg = "" if ddiff_register_volume is None else f"-v{ddiff_register_volume}:/var/lib/registry" + volume_arg = "" + if ddiff_register_volume is not None: + Path(ddiff_register_volume).mkdir(parents=True, exist_ok=True) + volume_arg = f"-v{ddiff_register_volume}:/var/lib/registry" tls_verify_flag = " --tls-verify=false" if container_runtime == "podman" else "" - cmd = f"{container_runtime} run -it -d -p{ddiff_port}:5000 {volume_arg} --name {ddiff_container_name}{tls_verify_flag} registry:2.8.3" + cmd = ( + f"{container_runtime} run -d -p{ddiff_port}:5000 {volume_arg} " + f"-e REGISTRY_STORAGE_DELETE_ENABLED=true " + f"--name {ddiff_container_name}{tls_verify_flag} registry:2.8.3" + ) run_command(cmd) + +def delete_image(tag): + prepared_tag = _prepare_tag(tag) + repo = prepared_tag.split(":", 1)[0] + digest = _request_manifest_digest(prepared_tag) + delete_url = f"{ddiff_url}/v2/{repo}/manifests/{digest}" + + req = urllib.request.Request(delete_url, method="DELETE") + try: + with urllib.request.urlopen(req) as res: + if res.status not in [202]: + print_error(f"Delete failed for {prepared_tag}: unexpected status {res.status} ({delete_url})") + print_debug(f"Deleted manifest for {prepared_tag} ({digest})") + except urllib.error.HTTPError as e: + print_error(f"Delete failed for {prepared_tag}: {e.code} {e.reason} ({delete_url})") + + config_path = "/etc/docker/registry/config.yml" + gc_cmd = f"{container_runtime} exec {ddiff_container_name} registry garbage-collect {config_path}" + run_command(gc_cmd) + print_debug("Registry garbage collection completed.") + def _skopeo_source_ref(host_tag, from_docker_hub): """Return the correct skopeo source transport:reference for Podman.""" # Podman stores short names under the localhost/ prefix in containers-storage @@ -301,7 +350,7 @@ def diff_image(base_tag, target_tag): print_debug(f"Done. Load the output image archive {archive_name} in the offline (ddiff load {archive_name})") print_debug(f"{archive_name}") -def load_image(base_tag, image_tarball): +def load_image(base_tag, image_tarball, delete_after_load=False): input_dir = ".ddiff-image" shutil.rmtree(input_dir, ignore_errors=True) with tarfile.open(image_tarball) as tar: @@ -358,6 +407,9 @@ def load_image(base_tag, image_tarball): else: print_debug(f"The image {target_tag} is sucessfully pulled on the host.") + if delete_after_load: + delete_image(target_tag) + def build_image(build_args): target_tag = None dockerfile_str = "Dockerfile" @@ -414,7 +466,7 @@ def list_blobs(tag): else: print_debug("Docker command not found. Switching to podman mode.") - if len(sys.argv) < 2 or not sys.argv[1] in ["server", "push", "pull", "diff", "load", "build", "list"]: + if len(sys.argv) < 2 or not sys.argv[1] in ["server", "push", "pull", "diff", "load", "build", "list", "delete"]: print("Usage: ddiff [command] [args...]") print("Commands:") print(" server - Run the registry server (set DDIFF_REGISTRY_VOLUME)") @@ -422,6 +474,7 @@ def list_blobs(tag): print(" pull ... - Pull one or more images") print(" diff - Diff the target image from the base image") print(" load () - Load the target image from diff file") + print(" delete - Delete an image from the registry and run GC") print(" build - Build the image and diff from base (FROM ...)") print(" list - List up blobs of the given image") sys.exit(1) @@ -450,13 +503,23 @@ def list_blobs(tag): sys.exit(1) diff_image(args[0], args[1]) elif command == "load": + delete_after_load = False + if "--delete" in args: + delete_after_load = True + args = [arg for arg in args if arg != "--delete"] + if len(args) > 2: - print("Usage: python3 ddiff.py load or python3 ddiff.py load ") + print("Usage: python3 ddiff.py load [--delete] or python3 ddiff.py load [--delete]") sys.exit(1) elif len(args) == 2: - load_image(args[0], args[1]) + load_image(args[0], args[1], delete_after_load=delete_after_load) elif len(args) == 1: - load_image(None, args[0]) + load_image(None, args[0], delete_after_load=delete_after_load) + elif command == "delete": + if len(args) != 1: + print("Usage: python3 ddiff.py delete ") + sys.exit(1) + delete_image(args[0]) elif command == "build": if len(args) < 1: print("Usage: python3 ddiff.py build ") diff --git a/test/test_delete_mode.sh b/test/test_delete_mode.sh new file mode 100755 index 0000000..594421b --- /dev/null +++ b/test/test_delete_mode.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DDIFF_PY="$ROOT_DIR/ddiff.py" +TMP_DIR="$ROOT_DIR/test/.tmp/delete-mode" +REGISTRY_DIR="$TMP_DIR/registry" +BASE_DOCKERFILE="$TMP_DIR/Dockerfile.base" +DELTA_DOCKERFILE="$TMP_DIR/Dockerfile.delta" + +PORT=5631 +REGISTRY_CONTAINER="ddiff-delete-registry" +BASE_TAG="ddiff-delete/base:latest" +TARGET_TAG="ddiff-delete/delta:latest" +ARCHIVE_PATH="$ROOT_DIR/ddiff-delete--delta-latest.tar.gz" + +log() { + echo "[test-delete] $*" +} + +wait_for_registry() { + for _ in $(seq 1 30); do + if curl -fsS "http://localhost:${PORT}/v2/" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +manifest_exists() { + local tag="$1" + local repo="${tag%%:*}" + local version="${tag##*:}" + curl -fsS -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ + "http://localhost:${PORT}/v2/${repo}/manifests/${version}" >/dev/null +} + +ensure_registry_image() { + local wanted="registry:2.8.3" + local candidate="" + + for ref in \ + "$wanted" \ + "docker.io/library/$wanted" \ + "docker.io/$wanted" \ + "localhost/$wanted"; do + if podman image exists "$ref" >/dev/null 2>&1; then + if [[ "$ref" != "$wanted" ]]; then + podman tag "$ref" "$wanted" >/dev/null + fi + return 0 + fi + done + + candidate="$(podman images --format '{{.Repository}}:{{.Tag}}' | awk '/(^|\/)registry:2\.8\.3$/ {print $1; exit}')" + if [[ -n "$candidate" ]]; then + podman tag "$candidate" "$wanted" >/dev/null + return 0 + fi + + if ! podman pull docker.io/library/$wanted >/dev/null 2>&1; then + log "SKIP: local image $wanted not found and pull is unavailable" + exit 0 + fi +} + +start_registry_or_skip() { + local log_file="$TMP_DIR/registry-start.log" + if ! DDIFF_FORCE_PODMAN=1 DDIFF_PORT="$PORT" DDIFF_CONTAINER_NAME="$REGISTRY_CONTAINER" DDIFF_REGISTRY_VOLUME="$REGISTRY_DIR" \ + python3 "$DDIFF_PY" server >"$log_file" 2>&1; then + if rg -q "setns: IO error: Operation not permitted" "$log_file"; then + log "SKIP: podman networking is not permitted in this environment" + cat "$log_file" + exit 0 + fi + cat "$log_file" + return 1 + fi +} + +cleanup() { + set +e + podman rm -f "$REGISTRY_CONTAINER" >/dev/null 2>&1 + rm -rf "$TMP_DIR" + rm -f "$ARCHIVE_PATH" + podman rmi "$TARGET_TAG" >/dev/null 2>&1 + podman rmi "$BASE_TAG" >/dev/null 2>&1 + set -e +} + +trap cleanup EXIT +cleanup +ensure_registry_image + +mkdir -p "$TMP_DIR" "$REGISTRY_DIR" +cat > "$BASE_DOCKERFILE" <<'DOCKER' +FROM scratch +ADD base.txt /base.txt +DOCKER +cat > "$TMP_DIR/base.txt" <<'TXT' +base layer +TXT +cat > "$DELTA_DOCKERFILE" <<'DOCKER' +FROM ddiff-delete/base:latest +ADD hello_ddiff.txt /hello_ddiff.txt +DOCKER +cat > "$TMP_DIR/hello_ddiff.txt" <<'TXT' +hello delete mode +TXT + +log "building local base and target images" +podman build -t "$BASE_TAG" -f "$BASE_DOCKERFILE" "$TMP_DIR" >/dev/null +podman build -t "$TARGET_TAG" -f "$DELTA_DOCKERFILE" "$TMP_DIR" >/dev/null + +log "starting ddiff registry server with delete enabled" +start_registry_or_skip +wait_for_registry + +log "creating diff archive from local images" +DDIFF_FORCE_PODMAN=1 DDIFF_PORT="$PORT" python3 - <<'PY' +import os, tarfile, shutil +import ddiff +base_tag = "ddiff-delete/base:latest" +target_tag = "ddiff-delete/delta:latest" + +ddiff.push_images([base_tag]) +ddiff.push_images([target_tag]) + +base_tag = ddiff._prepare_tag(base_tag) +target_tag = ddiff._prepare_tag(target_tag) +target_repo = target_tag.split(":")[0] + +output_dir = os.path.join(os.getcwd(), ".ddiff-image") +shutil.rmtree(output_dir, ignore_errors=True) +blob_dir = os.path.join(output_dir, "blobs") +os.makedirs(blob_dir) + +base_manifest, _ = ddiff._request_manifest(base_tag) +target_manifest, target_manifest_media_type = ddiff._request_manifest(target_tag) +base_blobs = ddiff._parse_blob_list(base_manifest) +target_blobs = ddiff._parse_blob_list(target_manifest) +diff_blobs = set(target_blobs) - set(base_blobs) + +for digest in diff_blobs: + ddiff._download_blob(target_repo, digest, blob_dir) + +with open(output_dir + "/manifest.json", "w") as f: + f.write(target_manifest) +with open(os.path.join(output_dir, "MANIFEST_MEDIA_TYPE"), "w") as f: + f.write(target_manifest_media_type) +with open(os.path.join(output_dir, "BASE"), "w") as f: + f.write(base_tag) +with open(os.path.join(output_dir, "TARGET"), "w") as f: + f.write(target_tag) +with open(os.path.join(output_dir, "MOUNT_BLOBS"), "w") as f: + f.write("|".join(list(set(target_blobs) - diff_blobs))) +with open(os.path.join(output_dir, "UPLOAD_BLOBS"), "w") as f: + f.write("|".join(diff_blobs)) + +archive_name = f"{target_tag.replace('/', '--').replace(':', '-')}.tar.gz" +with tarfile.open(archive_name, "w:gz") as tar: + tar.add(output_dir, arcname=".ddiff-image") +shutil.rmtree(output_dir) +PY +[[ -f "$ARCHIVE_PATH" ]] + +log "explicit delete command removes pushed target" +DDIFF_FORCE_PODMAN=1 DDIFF_PORT="$PORT" DDIFF_CONTAINER_NAME="$REGISTRY_CONTAINER" \ + python3 "$DDIFF_PY" delete "$TARGET_TAG" >/dev/null +if manifest_exists "$TARGET_TAG"; then + echo "manifest still exists after ddiff delete" + exit 1 +fi + +log "restart with clean registry data for load --delete scenario" +podman rm -f "$REGISTRY_CONTAINER" >/dev/null +rm -rf "$REGISTRY_DIR" +mkdir -p "$REGISTRY_DIR" +start_registry_or_skip +wait_for_registry + +log "load archive and delete from registry in one step" +DDIFF_FORCE_PODMAN=1 DDIFF_PORT="$PORT" DDIFF_CONTAINER_NAME="$REGISTRY_CONTAINER" \ + python3 "$DDIFF_PY" load "$BASE_TAG" "$ARCHIVE_PATH" --delete >/dev/null + +log "verify image loaded on host" +podman run --rm "$TARGET_TAG" cat /hello_ddiff.txt | grep -q 'hello delete mode' + +log "verify manifest no longer exists after load --delete" +if manifest_exists "$TARGET_TAG"; then + echo "manifest still exists after ddiff load --delete" + exit 1 +fi + +log "PASS"