Support new Apple Container

ktf · ktf · commit 36f066752518 · 2026-02-20T00:20:12.000+01:00
* No more need to have closed source Docker on Apple platforms
* Uses the Virtualization framework features of macOS Tahoe, in particular
  this should avoid having an always running VM
diff --git a/alibuild_helpers/args.py b/alibuild_helpers/args.py
@@ -430,9 +430,13 @@ def finaliseArgs(args, parser):
   if args.action in ("build", "doctor", "deps"):
     if args.dockerImage or args.docker_extra_args:
       args.docker = True
+    # In case we build with docker / containers, we add a special
+    # devel prefix (if not already present) so that we do not pollute
+    # the namespace of the current architecture
+    if args.docker and hasattr(args, "develPrefix"):
+      args.develPrefix = args.architecture
 
     args.docker_extra_args = shlex.split(args.docker_extra_args)
-    args.docker_extra_args.append("--network=host")
 
     if args.docker and args.architecture.startswith("osx"):
       parser.error("cannot use `-a %s` and --docker" % args.architecture)
diff --git a/alibuild_helpers/build.py b/alibuild_helpers/build.py
@@ -5,7 +5,7 @@
 from alibuild_helpers.analytics import report_event
 from alibuild_helpers.log import debug, info, banner, warning
 from alibuild_helpers.log import dieOnError
-from alibuild_helpers.cmd import execute, DockerRunner, BASH, install_wrapper_script, getstatusoutput
+from alibuild_helpers.cmd import execute, ContainerRunner, container_run_string, BASH, install_wrapper_script, getstatusoutput
 from alibuild_helpers.utilities import prunePaths, symlink, call_ignoring_oserrors, topological_sort, detectArch
 from alibuild_helpers.utilities import resolve_store_path
 from alibuild_helpers.utilities import parseDefaults, readDefaults
@@ -512,10 +512,10 @@ def doBuild(args, parser):
   }
   extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
 
-  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
+  with ContainerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_container:
     def performPreferCheckWithTempDir(pkg, cmd):
       with tempfile.TemporaryDirectory(prefix=f"alibuild_prefer_check_{pkg['package']}_") as temp_dir:
-        return getstatusoutput_docker(cmd, cwd=temp_dir)
+        return getstatusoutput_container(cmd, cwd=temp_dir)
 
     systemPackages, ownPackages, failed, validDefaults = \
       getPackageList(packages                = packages,
@@ -1091,28 +1091,9 @@ def performPreferCheckWithTempDir(pkg, cmd):
     # In case the --docker options is passed, we setup a docker container which
     # will perform the actual build. Otherwise build as usual using bash.
     if args.docker:
-      build_command = (
-        "docker run --rm --entrypoint= --user $(id -u):$(id -g) "
-        "-v {workdir}:/sw -v{configDir}:/alidist:ro -v {scriptDir}/build.sh:/build.sh:ro "
-        "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} "
-        "-e WORK_DIR_OVERRIDE=/sw -e ALIBUILD_CONFIG_DIR_OVERRIDE=/alidist {extraArgs} {image} bash -ex /build.sh"
-      ).format(
-        image=quote(args.dockerImage),
-        workdir=quote(abspath(args.workDir)),
-        configDir=quote(abspath(args.configDir)),
-        scriptDir=quote(scriptDir),
-        extraArgs=" ".join(map(quote, args.docker_extra_args)),
-        additionalEnv=" ".join(
-          f"-e {var}={quote(value)}" for var, value in buildEnvironment),
-        # Used e.g. by O2DPG-sim-tests to find the O2DPG repository.
-        develVolumes=" ".join(
-          '-v "$PWD/$(readlink {pkg} || echo {pkg})":/{pkg}:rw'.format(pkg=quote(spec["package"]))
-          for spec in specs.values() if spec["is_devel_pkg"]),
-        additionalVolumes=" ".join(
-          "-v %s" % quote(volume) for volume in args.volumes),
-        mirrorVolume=("-v %s:/mirror" % quote(dirname(spec["reference"]))
-                      if "reference" in spec else ""),
-      )
+      build_command = container_run_string(args.dockerImage, args.workDir, args.configDir, scriptDir,
+                                           args.docker_extra_args, spec, specs, args.volumes, buildEnvironment)
+
     else:
       os.environ.update(buildEnvironment)
       build_command = f"{BASH} -e -x {quote(scriptDir)}/build.sh 2>&1"
diff --git a/alibuild_helpers/build_template.sh b/alibuild_helpers/build_template.sh
@@ -293,7 +293,7 @@ mkdir -p "${WORK_DIR}/TARS/$HASH_PATH" \
 PACKAGE_WITH_REV=$PKGNAME-$PKGVERSION-$PKGREVISION.$ARCHITECTURE.tar.gz
 # Copy and tar/compress (if applicable) in parallel.
 # Use -H to match tar's behaviour of preserving hardlinks.
-rsync -aH "$WORK_DIR/INSTALLROOT/$PKGHASH/" "$WORK_DIR" & rsync_pid=$!
+rsync -DgloprH "$WORK_DIR/INSTALLROOT/$PKGHASH/" "$WORK_DIR" & rsync_pid=$!
 if [ "$CAN_DELETE" = 1 ]; then
   # We're deleting the tarball anyway, so no point in creating a new one.
   # There might be an old existing tarball, and we should delete it.
diff --git a/alibuild_helpers/cmd.py b/alibuild_helpers/cmd.py
@@ -5,6 +5,7 @@
 from textwrap import dedent
 from subprocess import TimeoutExpired
 from shlex import quote
+import platform
 
 from alibuild_helpers.log import debug, error, dieOnError
 
@@ -70,6 +71,52 @@ def execute(command, printer=debug, timeout=None):
 
 BASH = "bash" if getstatusoutput("/bin/bash --version")[0] else "/bin/bash"
 
+class AppleContainerRunner:
+    """A context manager for running commands inside a Apple Container (see https://github.com/apple/container)
+       If the image given is None or empty, the commands are run on the host instead.
+    """
+    def __init__(self, container_image, container_run_args, extra_env, extra_volumes) -> None:
+        self._container_image = container_image
+        self._container_run_args = container_run_args
+        self._container = None
+        self._extra_env = extra_env
+        self._extra_volumes = extra_volumes
+
+    def __enter__(self):
+        def getstatusoutput_host(cmd, cwd=None):
+            command_prefix=""
+            if self._extra_env:
+                command_prefix="env " + " ".join("{}={}".format(k, quote(v)) for (k,v) in self._extra_env.items()) + " "
+            return getstatusoutput("{}{} -c {}".format(command_prefix, BASH, quote(cmd))
+                                , cwd=cwd)
+
+        if not self._container_image:
+            return getstatusoutput_host
+
+        envOpts = [opt for k, v in self._extra_env.items() for opt in ("-e", f"{k}={v}")]
+        volumes = [opt for v in self._extra_volumes for opt in ("-v", v)]
+        # Apple Container is more picky about missing entrypoints, so we always override it
+        # with /bin/sleep.
+        cmd = ["container", "run", "--detach"] + envOpts + volumes + ["--rm", "--entrypoint=/bin/sleep"]
+        cmd += self._container_run_args
+        cmd += [self._container_image, "inf"]
+        self._container = getoutput(cmd).strip()
+
+        def getstatusoutput_container(cmd, cwd=None):
+            if self._container is None:
+                return getstatusoutput_host(cmd, cwd=cwd)
+            envOpts = [opt for k, v in self._extra_env.items() for opt in ("-e", f"{k}={v}")]
+            exec_cmd = ["container", "exec"] + envOpts + [self._container, "bash", "-c", cmd]
+            return getstatusoutput(exec_cmd, cwd=cwd)
+
+        return getstatusoutput_container
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self._container is not None:
+            getstatusoutput("container kill " + quote(self._container))
+        self._container = None
+        return False  # propagate any exception that may have occurred
+
 
 class DockerRunner:
   """A context manager for running commands inside a Docker container.
@@ -146,3 +193,85 @@ def install_wrapper_script(name, work_dir):
              "rerunning this command inside a login shell (e.g. `bash -l`). "
              "If that doesn't work, run `export PATH` manually.")
   os.environ["PATH"] = script_dir + ":" + os.environ["PATH"]
+
+def to_int(s: str) -> int:
+    try:
+        return int(float(s))
+    except (ValueError, TypeError):
+        return 0
+
+
+def _apple_run_string(dockerImage, workDir, configDir, scriptDir, docker_extra_args, spec, specs, volumes, buildEnvironment):
+    build_command = (
+        "container run --rm --entrypoint=/bin/bash --user $(id -u):$(id -g) "
+        "--mount type=bind,source={workdir},target=/sw "
+        "--mount type=bind,source={configDir},target=/alidist,readonly "
+        "--mount type=bind,source={scriptDir},target=/scripts,readonly "
+        "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} "
+        "-e WORK_DIR_OVERRIDE=/sw -e ALIBUILD_CONFIG_DIR_OVERRIDE=/alidist {extraArgs} {image} -ex /scripts/build.sh"
+    ).format(
+        image=quote(dockerImage),
+        workdir=quote(os.path.abspath(workDir)),
+        configDir=quote(os.path.abspath(configDir)),
+        scriptDir=quote(scriptDir),
+        extraArgs=" ".join(map(quote, docker_extra_args)),
+        additionalEnv=" ".join(
+            "-e {}={}".format(var, quote(value)) for var, value in buildEnvironment
+        ),
+        # Used e.g. by O2DPG-sim-tests to find the O2DPG repository.
+        develVolumes=" ".join(
+            '--mount type=bind,source="$PWD/$(readlink {pkg} || echo {pkg})",target=/{pkg},readonly'.format(
+                pkg=quote(s["package"])
+            )
+            for s in specs.values()
+            if s["is_devel_pkg"]
+        ),
+        additionalVolumes=" ".join("--mount type=bind,source=%s" % quote(volume) for volume in volumes),
+        mirrorVolume=(
+            "--mount source=%s,target=/mirror" % quote(os.path.dirname(spec["reference"]))
+            if "reference" in spec
+            else ""
+        ),
+    )
+    print(build_command)
+    return build_command
+
+def _docker_run_string( dockerImage, workDir, configDir, scriptDir, docker_extra_args, spec, specs, volumes, buildEnvironment):
+    build_command = (
+        "docker run --rm --entrypoint=/bin/bash --user $(id -u):$(id -g) "
+        "-v {workdir}:/sw -v{configDir}:/alidist:ro -v {scriptDir}/build.sh:/build.sh:ro "
+        "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} "
+        "-e WORK_DIR_OVERRIDE=/sw -e ALIBUILD_CONFIG_DIR_OVERRIDE=/alidist {extraArgs} --network=host {image} bash -ex /build.sh"
+    ).format(
+        image=quote(dockerImage),
+        workdir=quote(os.path.abspath(workDir)),
+        configDir=quote(os.path.abspath(configDir)),
+        scriptDir=quote(scriptDir),
+        extraArgs=" ".join(map(quote, docker_extra_args)),
+        additionalEnv=" ".join(
+            "-e {}={}".format(var, quote(value)) for var, value in buildEnvironment
+        ),
+        # Used e.g. by O2DPG-sim-tests to find the O2DPG repository.
+        develVolumes=" ".join(
+            '-v "$PWD/$(readlink {pkg} || echo {pkg})":/{pkg}:rw'.format(
+                pkg=quote(spec["package"])
+            )
+            for spec in specs.values()
+            if spec["is_devel_pkg"]
+        ),
+        additionalVolumes=" ".join("-v %s" % quote(volume) for volume in volumes),
+        mirrorVolume=(
+            "-v %s:/mirror" % quote(os.path.dirname(spec["reference"]))
+            if "reference" in spec
+            else ""
+        ),
+    )
+    return build_command
+
+
+if to_int(platform.mac_ver()[0].split(".")[0]) >= 26:
+    ContainerRunner = AppleContainerRunner
+    container_run_string = _apple_run_string
+else:
+    ContainerRunner = DockerRunner
+    container_run_string = _docker_run_string
diff --git a/alibuild_helpers/deps.py b/alibuild_helpers/deps.py
@@ -2,7 +2,7 @@
 
 from alibuild_helpers.log import debug, error, info, dieOnError
 from alibuild_helpers.utilities import parseDefaults, readDefaults, getPackageList, validateDefaults
-from alibuild_helpers.cmd import DockerRunner, execute
+from alibuild_helpers.cmd import ContainerRunner, execute
 from tempfile import NamedTemporaryFile
 from os import remove, path
 
@@ -20,7 +20,7 @@ def doDeps(args, parser):
   extra_env = {"ALIBUILD_CONFIG_DIR": "/alidist" if args.docker else path.abspath(args.configDir)}
   extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
   
-  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
+  with ContainerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
     def performCheck(pkg, cmd):
       return getstatusoutput_docker(cmd)
     
diff --git a/alibuild_helpers/doctor.py b/alibuild_helpers/doctor.py
@@ -7,7 +7,7 @@
 from alibuild_helpers.log import debug, error, banner, info, success, warning
 from alibuild_helpers.log import logger
 from alibuild_helpers.utilities import getPackageList, parseDefaults, readDefaults, validateDefaults
-from alibuild_helpers.cmd import getstatusoutput, DockerRunner
+from alibuild_helpers.cmd import getstatusoutput, ContainerRunner
 import tempfile
 
 def prunePaths(workDir) -> None:
@@ -89,7 +89,7 @@ def doDoctor(args, parser):
   extra_env = {"ALIBUILD_CONFIG_DIR": "/alidist" if args.docker else os.path.abspath(args.configDir)}
   extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
 
-  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
+  with ContainerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
     err, output = getstatusoutput_docker("type c++")
   if err:
     warning("Unable to find system compiler.\n"
@@ -149,7 +149,7 @@ def performValidateDefaults(spec):
   extra_env = {"ALIBUILD_CONFIG_DIR": "/alidist" if args.docker else os.path.abspath(args.configDir)}
   extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
   
-  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
+  with ContainerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
     fromSystem, own, failed, validDefaults = \
       getPackageList(packages                = packages,
                      specs                   = specs,
diff --git a/tests/test_cmd.py b/tests/test_cmd.py
@@ -1,7 +1,7 @@
 # Assuming you are using the mock library to ... mock things
 from unittest import mock
 
-from alibuild_helpers.cmd import execute, DockerRunner
+from alibuild_helpers.cmd import execute, ContainerRunner
 
 import unittest
 
@@ -20,9 +20,9 @@ def test_execute(self, mock_debug):
 
     @mock.patch("alibuild_helpers.cmd.getoutput")
     @mock.patch("alibuild_helpers.cmd.getstatusoutput")
-    def test_DockerRunner(self, mock_getstatusoutput, mock_getoutput):
+    def test_ContainerRunner(self, mock_getstatusoutput, mock_getoutput):
         mock_getoutput.side_effect = lambda cmd: "container-id\n"
-        with DockerRunner("image", ["extra arg"]) as getstatusoutput_docker:
+        with ContainerRunner("image", ["extra arg"]) as getstatusoutput_docker:
             mock_getoutput.assert_called_with(["docker", "run", "--detach", "--rm", "--entrypoint=",
                                                "extra arg", "image", "sleep", "inf"])
             getstatusoutput_docker("echo foo")
@@ -31,7 +31,7 @@ def test_DockerRunner(self, mock_getstatusoutput, mock_getoutput):
 
         mock_getoutput.reset_mock()
         mock_getstatusoutput.reset_mock()
-        with DockerRunner("") as getstatusoutput_docker:
+        with ContainerRunner("") as getstatusoutput_docker:
             mock_getoutput.assert_not_called()
             getstatusoutput_docker("echo foo")
             mock_getstatusoutput.assert_called_with("/bin/bash -c 'echo foo'", cwd=None)
@@ -40,13 +40,13 @@ def test_DockerRunner(self, mock_getstatusoutput, mock_getoutput):
 
     @mock.patch("alibuild_helpers.cmd.getoutput")
     @mock.patch("alibuild_helpers.cmd.getstatusoutput")
-    def test_DockerRunner_with_env_vars(self, mock_getstatusoutput, mock_getoutput):
+    def test_ContainerRunner_with_env_vars(self, mock_getstatusoutput, mock_getoutput):
         # Test that environment variables are properly injected into docker exec commands.
         mock_getoutput.side_effect = lambda cmd: "container-id\n"
         
         # Test with environment variables
         extra_env = {"TEST_VAR": "test_value", "ANOTHER_VAR": "another_value"}
-        with DockerRunner("image", extra_env=extra_env) as getstatusoutput_docker:
+        with ContainerRunner("image", extra_env=extra_env) as getstatusoutput_docker:
             # Verify container creation includes environment variables
             mock_getoutput.assert_called_with(["docker", "run", "--detach", 
                                                "-e", "TEST_VAR=test_value",
@@ -63,29 +63,29 @@ def test_DockerRunner_with_env_vars(self, mock_getstatusoutput, mock_getoutput):
         # Test host execution with environment variables
         mock_getoutput.reset_mock()
         mock_getstatusoutput.reset_mock()
-        with DockerRunner("", extra_env=extra_env) as getstatusoutput_docker:
+        with ContainerRunner("", extra_env=extra_env) as getstatusoutput_docker:
             mock_getoutput.assert_not_called()
             getstatusoutput_docker("echo test")
             mock_getstatusoutput.assert_called_with("env TEST_VAR=test_value ANOTHER_VAR=another_value /bin/bash -c 'echo test'", cwd=None)
 
     @mock.patch("alibuild_helpers.cmd.getoutput")
     @mock.patch("alibuild_helpers.cmd.getstatusoutput")
-    def test_DockerRunner_multiline_env_var(self, mock_getstatusoutput, mock_getoutput):
+    def test_ContainerRunner_multiline_env_var(self, mock_getstatusoutput, mock_getoutput):
         multiline_value = "line1\nline2\nline3"
         extra_env = {"MULTILINE_VAR": multiline_value}
         
-        with DockerRunner("", extra_env=extra_env) as getstatusoutput_docker:
+        with ContainerRunner("", extra_env=extra_env) as getstatusoutput_docker:
             mock_getoutput.assert_not_called()
             getstatusoutput_docker("echo test")
             mock_getstatusoutput.assert_called_with("env MULTILINE_VAR='line1\nline2\nline3' /bin/bash -c 'echo test'", cwd=None)
 
     @mock.patch("alibuild_helpers.cmd.getoutput")
     @mock.patch("alibuild_helpers.cmd.getstatusoutput")
-    def test_DockerRunner_env_var_with_semicolon(self, mock_getstatusoutput, mock_getoutput):
+    def test_ContainerRunner_env_var_with_semicolon(self, mock_getstatusoutput, mock_getoutput):
         semicolon_value = "value1;value2;value3"
         extra_env = {"SEMICOLON_VAR": semicolon_value}
         
-        with DockerRunner("", extra_env=extra_env) as getstatusoutput_docker:
+        with ContainerRunner("", extra_env=extra_env) as getstatusoutput_docker:
             mock_getoutput.assert_not_called()
             getstatusoutput_docker("echo test")
             mock_getstatusoutput.assert_called_with("env SEMICOLON_VAR='value1;value2;value3' /bin/bash -c 'echo test'", cwd=None)
