Harden delete-mode validation and registry startup behavior

README.md

@@ -65,12 +65,13 @@ source ~/.bashrc
 **Usage**: `ddiff <command> <args...>`
 
 **Commands**
-  - `server`                      - Run the registry server
+  - `server`                      - Run the registry server (delete API + GC enabled)
   - `push` `<tag 1> ... <tag n>`   - Push one or more images
   - `pull` `<tag 1> ... <tag n>`    - Pull one or more images
   - `diff` `<base> <target>`       - Diff the target image from the base image
-  - `load` `<tar file>`             - Load the target image from diff file
+  - `load` `(<base tag>) <tar file> [--delete]` - Load the target image from diff file and optionally delete it from registry
   - `build` `<args>`                - Build the image and diff from base (FROM ...)
+  - `delete` `<tag>`                 - Delete one image tag from the registry and run garbage collection
 
 # Docker base images
 For easier sharing of base images, DockerDiff provides several pre-configured base images. 

ddiff.py

@@ -110,6 +110,27 @@ def _request_manifest(tag):
         print_error(f"HTTP error: {e.code} - {e.reason} ({manifest_url})")
     except Exception as e:
         print_error(f"Error: {e} ({manifest_url})")
+
+
+def _request_manifest_digest(tag):
+    if "@" in tag:
+        repo, reference = tag.split("@", 1)
+    else:
+        repo, reference = tag.split(":", 1)
+    manifest_url = f"{ddiff_url}/v2/{repo}/manifests/{reference}"
+
+    req = urllib.request.Request(manifest_url)
+    req.add_header("Accept", ACCEPT_MANIFEST_TYPES)
+    try:
+        with urllib.request.urlopen(req) as response:
+            digest = response.getheader("Docker-Content-Digest")
+            if not digest:
+                print_error(f"Could not resolve manifest digest from response headers ({manifest_url})")
+            return digest
+    except urllib.error.HTTPError as e:
+        print_error(f"HTTP error: {e.code} - {e.reason} ({manifest_url})")
+    except Exception as e:
+        print_error(f"Error: {e} ({manifest_url})")
 
 def _validate_manifest_media_type(manifest):
     media_type = manifest.get("mediaType", "")
@@ -201,12 +222,40 @@ def _upload_manifest(tag, manifest_path, manifest_media_type=DOCKER_MANIFEST_V2)
         print_error(f"Manifest upload failed: {e.code} {e.reason} ({url})")
 
 def run_registry():
-    volume_arg = "" if ddiff_register_volume is None else f"-v{ddiff_register_volume}:/var/lib/registry"
+    volume_arg = ""
+    if ddiff_register_volume is not None:
+        Path(ddiff_register_volume).mkdir(parents=True, exist_ok=True)
+        volume_arg = f"-v{ddiff_register_volume}:/var/lib/registry"
 
     tls_verify_flag = " --tls-verify=false" if container_runtime == "podman" else ""
-    cmd = f"{container_runtime} run -it -d -p{ddiff_port}:5000 {volume_arg} --name {ddiff_container_name}{tls_verify_flag} registry:2.8.3"
+    cmd = (
+        f"{container_runtime} run -d -p{ddiff_port}:5000 {volume_arg} "
+        f"-e REGISTRY_STORAGE_DELETE_ENABLED=true "
+        f"--name {ddiff_container_name}{tls_verify_flag} registry:2.8.3"
+    )
     run_command(cmd)
 
+
+def delete_image(tag):
+    prepared_tag = _prepare_tag(tag)
+    repo = prepared_tag.split(":", 1)[0]
+    digest = _request_manifest_digest(prepared_tag)
+    delete_url = f"{ddiff_url}/v2/{repo}/manifests/{digest}"
+
+    req = urllib.request.Request(delete_url, method="DELETE")
+    try:
+        with urllib.request.urlopen(req) as res:
+            if res.status not in [202]:
+                print_error(f"Delete failed for {prepared_tag}: unexpected status {res.status} ({delete_url})")
+            print_debug(f"Deleted manifest for {prepared_tag} ({digest})")
+    except urllib.error.HTTPError as e:
+        print_error(f"Delete failed for {prepared_tag}: {e.code} {e.reason} ({delete_url})")
+
+    config_path = "/etc/docker/registry/config.yml"
+    gc_cmd = f"{container_runtime} exec {ddiff_container_name} registry garbage-collect {config_path}"
+    run_command(gc_cmd)
+    print_debug("Registry garbage collection completed.")
+
 def _skopeo_source_ref(host_tag, from_docker_hub):
     """Return the correct skopeo source transport:reference for Podman."""
     # Podman stores short names under the localhost/ prefix in containers-storage
@@ -301,7 +350,7 @@ def diff_image(base_tag, target_tag):
     print_debug(f"Done. Load the output image archive {archive_name} in the offline (ddiff load {archive_name})")
     print_debug(f"{archive_name}")
 
-def load_image(base_tag, image_tarball):
+def load_image(base_tag, image_tarball, delete_after_load=False):
     input_dir = ".ddiff-image"
     shutil.rmtree(input_dir, ignore_errors=True)
     with tarfile.open(image_tarball) as tar:
@@ -358,6 +407,9 @@ def load_image(base_tag, image_tarball):
     else:
         print_debug(f"The image {target_tag} is sucessfully pulled on the host.")
 
+    if delete_after_load:
+        delete_image(target_tag)
+
 def build_image(build_args):
     target_tag = None
     dockerfile_str = "Dockerfile"
@@ -414,14 +466,15 @@ def list_blobs(tag):
         else:
             print_debug("Docker command not found. Switching to podman mode.")
 
-    if len(sys.argv) < 2 or not sys.argv[1] in ["server", "push", "pull", "diff", "load", "build", "list"]:
+    if len(sys.argv) < 2 or not sys.argv[1] in ["server", "push", "pull", "diff", "load", "build", "list", "delete"]:
         print("Usage: ddiff [command] [args...]")
         print("Commands:")
         print("  server                       - Run the registry server (set DDIFF_REGISTRY_VOLUME)")
         print("  push <tag 1> ... <tag n>     - Push one or more images")
         print("  pull <tag 1> ... <tag n>     - Pull one or more images")
         print("  diff <base> <target>         - Diff the target image from the base image")
         print("  load (<base tag>) <tar file> - Load the target image from diff file")
+        print("  delete <tag>                 - Delete an image from the registry and run GC")
         print("  build <args>                 - Build the image and diff from base (FROM ...)")
         print("  list <tag>                   - List up blobs of the given image")
         sys.exit(1)
@@ -450,13 +503,23 @@ def list_blobs(tag):
             sys.exit(1)
         diff_image(args[0], args[1])
     elif command == "load":
+        delete_after_load = False
+        if "--delete" in args:
+            delete_after_load = True
+            args = [arg for arg in args if arg != "--delete"]
+
         if len(args) > 2:
-            print("Usage: python3 ddiff.py load <base tag> <tar_file> or python3 ddiff.py load <tar_file>")
+            print("Usage: python3 ddiff.py load <base tag> <tar_file> [--delete] or python3 ddiff.py load <tar_file> [--delete]")
             sys.exit(1)
         elif len(args) == 2:
-            load_image(args[0], args[1])
+            load_image(args[0], args[1], delete_after_load=delete_after_load)
         elif len(args) == 1:
-            load_image(None, args[0])
+            load_image(None, args[0], delete_after_load=delete_after_load)
+    elif command == "delete":
+        if len(args) != 1:
+            print("Usage: python3 ddiff.py delete <tag>")
+            sys.exit(1)
+        delete_image(args[0])
     elif command == "build":
         if len(args) < 1:
             print("Usage: python3 ddiff.py build <docker build args>")

test/test_delete_mode.sh

@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+DDIFF_PY="$ROOT_DIR/ddiff.py"
+TMP_DIR="$ROOT_DIR/test/.tmp/delete-mode"
+REGISTRY_DIR="$TMP_DIR/registry"
+BASE_DOCKERFILE="$TMP_DIR/Dockerfile.base"
+DELTA_DOCKERFILE="$TMP_DIR/Dockerfile.delta"
+
+PORT=5631
+REGISTRY_CONTAINER="ddiff-delete-registry"
+BASE_TAG="ddiff-delete/base:latest"
+TARGET_TAG="ddiff-delete/delta:latest"
+ARCHIVE_PATH="$ROOT_DIR/ddiff-delete--delta-latest.tar.gz"
+
+log() {
+  echo "[test-delete] $*"
+}
+
+wait_for_registry() {
+  for _ in $(seq 1 30); do
+    if curl -fsS "http://localhost:${PORT}/v2/" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 1
+  done
+  return 1
+}
+
+manifest_exists() {
+  local tag="$1"
+  local repo="${tag%%:*}"
+  local version="${tag##*:}"
+  curl -fsS -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
+    "http://localhost:${PORT}/v2/${repo}/manifests/${version}" >/dev/null
+}
+
+ensure_registry_image() {
+  local wanted="registry:2.8.3"
+  local candidate=""
+
+  for ref in \
+    "$wanted" \
+    "docker.io/library/$wanted" \
+    "docker.io/$wanted" \
+    "localhost/$wanted"; do
+    if podman image exists "$ref" >/dev/null 2>&1; then
+      if [[ "$ref" != "$wanted" ]]; then
+        podman tag "$ref" "$wanted" >/dev/null
+      fi
+      return 0
+    fi
+  done
+
+  candidate="$(podman images --format '{{.Repository}}:{{.Tag}}' | awk '/(^|\/)registry:2\.8\.3$/ {print $1; exit}')"
+  if [[ -n "$candidate" ]]; then
+    podman tag "$candidate" "$wanted" >/dev/null
+    return 0
+  fi
+
+  if ! podman pull docker.io/library/$wanted >/dev/null 2>&1; then
+    log "SKIP: local image $wanted not found and pull is unavailable"
+    exit 0
+  fi
+}
+
+start_registry_or_skip() {
+  local log_file="$TMP_DIR/registry-start.log"
+  if ! DDIFF_FORCE_PODMAN=1 DDIFF_PORT="$PORT" DDIFF_CONTAINER_NAME="$REGISTRY_CONTAINER" DDIFF_REGISTRY_VOLUME="$REGISTRY_DIR" \
+      python3 "$DDIFF_PY" server >"$log_file" 2>&1; then
+    if rg -q "setns: IO error: Operation not permitted" "$log_file"; then
+      log "SKIP: podman networking is not permitted in this environment"
+      cat "$log_file"
+      exit 0
+    fi
+    cat "$log_file"
+    return 1
+  fi
+}
+
+cleanup() {
+  set +e
+  podman rm -f "$REGISTRY_CONTAINER" >/dev/null 2>&1
+  rm -rf "$TMP_DIR"
+  rm -f "$ARCHIVE_PATH"
+  podman rmi "$TARGET_TAG" >/dev/null 2>&1
+  podman rmi "$BASE_TAG" >/dev/null 2>&1
+  set -e
+}
+
+trap cleanup EXIT
+cleanup
+ensure_registry_image
+
+mkdir -p "$TMP_DIR" "$REGISTRY_DIR"
+cat > "$BASE_DOCKERFILE" <<'DOCKER'
+FROM scratch
+ADD base.txt /base.txt
+DOCKER
+cat > "$TMP_DIR/base.txt" <<'TXT'
+base layer
+TXT
+cat > "$DELTA_DOCKERFILE" <<'DOCKER'
+FROM ddiff-delete/base:latest
+ADD hello_ddiff.txt /hello_ddiff.txt
+DOCKER
+cat > "$TMP_DIR/hello_ddiff.txt" <<'TXT'
+hello delete mode
+TXT
+
+log "building local base and target images"
+podman build -t "$BASE_TAG" -f "$BASE_DOCKERFILE" "$TMP_DIR" >/dev/null
+podman build -t "$TARGET_TAG" -f "$DELTA_DOCKERFILE" "$TMP_DIR" >/dev/null
+
+log "starting ddiff registry server with delete enabled"
+start_registry_or_skip
+wait_for_registry
+
+log "creating diff archive from local images"
+DDIFF_FORCE_PODMAN=1 DDIFF_PORT="$PORT" python3 - <<'PY'
+import os, tarfile, shutil
+import ddiff
+base_tag = "ddiff-delete/base:latest"
+target_tag = "ddiff-delete/delta:latest"
+
+ddiff.push_images([base_tag])
+ddiff.push_images([target_tag])
+
+base_tag = ddiff._prepare_tag(base_tag)
+target_tag = ddiff._prepare_tag(target_tag)
+target_repo = target_tag.split(":")[0]
+
+output_dir = os.path.join(os.getcwd(), ".ddiff-image")
+shutil.rmtree(output_dir, ignore_errors=True)
+blob_dir = os.path.join(output_dir, "blobs")
+os.makedirs(blob_dir)
+
+base_manifest, _ = ddiff._request_manifest(base_tag)
+target_manifest, target_manifest_media_type = ddiff._request_manifest(target_tag)
+base_blobs = ddiff._parse_blob_list(base_manifest)
+target_blobs = ddiff._parse_blob_list(target_manifest)
+diff_blobs = set(target_blobs) - set(base_blobs)
+
+for digest in diff_blobs:
+    ddiff._download_blob(target_repo, digest, blob_dir)
+
+with open(output_dir + "/manifest.json", "w") as f:
+    f.write(target_manifest)
+with open(os.path.join(output_dir, "MANIFEST_MEDIA_TYPE"), "w") as f:
+    f.write(target_manifest_media_type)
+with open(os.path.join(output_dir, "BASE"), "w") as f:
+    f.write(base_tag)
+with open(os.path.join(output_dir, "TARGET"), "w") as f:
+    f.write(target_tag)
+with open(os.path.join(output_dir, "MOUNT_BLOBS"), "w") as f:
+    f.write("|".join(list(set(target_blobs) - diff_blobs)))
+with open(os.path.join(output_dir, "UPLOAD_BLOBS"), "w") as f:
+    f.write("|".join(diff_blobs))
+
+archive_name = f"{target_tag.replace('/', '--').replace(':', '-')}.tar.gz"
+with tarfile.open(archive_name, "w:gz") as tar:
+    tar.add(output_dir, arcname=".ddiff-image")
+shutil.rmtree(output_dir)
+PY
+[[ -f "$ARCHIVE_PATH" ]]
+
+log "explicit delete command removes pushed target"
+DDIFF_FORCE_PODMAN=1 DDIFF_PORT="$PORT" DDIFF_CONTAINER_NAME="$REGISTRY_CONTAINER" \
+  python3 "$DDIFF_PY" delete "$TARGET_TAG" >/dev/null
+if manifest_exists "$TARGET_TAG"; then
+  echo "manifest still exists after ddiff delete"
+  exit 1
+fi
+
+log "restart with clean registry data for load --delete scenario"
+podman rm -f "$REGISTRY_CONTAINER" >/dev/null
+rm -rf "$REGISTRY_DIR"
+mkdir -p "$REGISTRY_DIR"
+start_registry_or_skip
+wait_for_registry
+
+log "load archive and delete from registry in one step"
+DDIFF_FORCE_PODMAN=1 DDIFF_PORT="$PORT" DDIFF_CONTAINER_NAME="$REGISTRY_CONTAINER" \
+  python3 "$DDIFF_PY" load "$BASE_TAG" "$ARCHIVE_PATH" --delete >/dev/null
+
+log "verify image loaded on host"
+podman run --rm "$TARGET_TAG" cat /hello_ddiff.txt | grep -q 'hello delete mode'
+
+log "verify manifest no longer exists after load --delete"
+if manifest_exists "$TARGET_TAG"; then
+  echo "manifest still exists after ddiff load --delete"
+  exit 1
+fi
+
+log "PASS"