Support new Apple Container

alibuild_helpers/args.py

@@ -428,21 +428,27 @@ def finaliseArgs(args, parser):
   if args.action in ("build", "doctor", "deps"):
     if args.dockerImage or args.docker_extra_args:
       args.docker = True
+    # In case we build with docker / containers, we add a special
+    # devel prefix (if not already present) so that we do not pollute
+    # the namespace of the current architecture
+    if args.docker and getattr(args, "develPrefix", None):
+      args.develPrefix = args.architecture
 
     args.docker_extra_args = shlex.split(args.docker_extra_args)
-    args.docker_extra_args.append("--network=host")
 
     if args.docker and args.architecture.startswith("osx"):
       parser.error("cannot use `-a %s` and --docker" % args.architecture)
 
-    if args.docker and commands.getstatusoutput("which docker")[0]:
-      parser.error("cannot use --docker as docker executable is not found")
+    if args.docker and commands.getstatusoutput("which docker")[0] and commands.getstatusoutput("which container")[0]:
+      parser.error("cannot use --docker as no container runtime (docker or container) was found")
 
     # If specified, used the docker image requested, otherwise, if running
     # in docker the docker image is given by the first part of the
     # architecture we want to build for.
     if args.docker and not args.dockerImage:
-      args.dockerImage = "registry.cern.ch/alisw/%s-builder" % args.architecture.split("_")[0]
+      distro = args.architecture.split("_")[0]
+      cpu_suffix = "-arm" if args.architecture.endswith("_aarch64") else ""
+      args.dockerImage = "registry.cern.ch/alisw/%s%s-builder" % (distro, cpu_suffix)
 
   if "annotate" in args:
     for comment_assignment in args.annotate:

alibuild_helpers/build.py

@@ -5,7 +5,7 @@
 from alibuild_helpers.analytics import report_event
 from alibuild_helpers.log import debug, info, banner, warning
 from alibuild_helpers.log import dieOnError
-from alibuild_helpers.cmd import execute, DockerRunner, BASH, install_wrapper_script, getstatusoutput
+from alibuild_helpers.cmd import execute, ContainerRunner, container_run_string, BASH, install_wrapper_script, getstatusoutput
 from alibuild_helpers.utilities import pruneWorkdirFromPaths, pruneVersionEnvVars, symlink, call_ignoring_oserrors, topological_sort, detectArch
 from alibuild_helpers.utilities import resolve_store_path
 from alibuild_helpers.utilities import parseDefaults, readDefaults
@@ -512,10 +512,10 @@ def doBuild(args, parser):
   }
   extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
 
-  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
+  with ContainerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_container:
     def performPreferCheckWithTempDir(pkg, cmd):
       with tempfile.TemporaryDirectory(prefix=f"alibuild_prefer_check_{pkg['package']}_") as temp_dir:
-        return getstatusoutput_docker(cmd, cwd=temp_dir)
+        return getstatusoutput_container(cmd, cwd=temp_dir)
 
     systemPackages, ownPackages, failed, validDefaults = \
       getPackageList(packages                = packages,
@@ -1093,28 +1093,9 @@ def performPreferCheckWithTempDir(pkg, cmd):
     # In case the --docker options is passed, we setup a docker container which
     # will perform the actual build. Otherwise build as usual using bash.
     if args.docker:
-      build_command = (
-        "docker run --rm --entrypoint= --user $(id -u):$(id -g) "
-        "-v {workdir}:/sw -v{configDir}:/alidist:ro -v {scriptDir}/build.sh:/build.sh:ro "
-        "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} "
-        "-e WORK_DIR_OVERRIDE=/sw -e ALIBUILD_CONFIG_DIR_OVERRIDE=/alidist {extraArgs} {image} bash -ex /build.sh"
-      ).format(
-        image=quote(args.dockerImage),
-        workdir=quote(abspath(args.workDir)),
-        configDir=quote(abspath(args.configDir)),
-        scriptDir=quote(scriptDir),
-        extraArgs=" ".join(map(quote, args.docker_extra_args)),
-        additionalEnv=" ".join(
-          f"-e {var}={quote(value)}" for var, value in buildEnvironment),
-        # Used e.g. by O2DPG-sim-tests to find the O2DPG repository.
-        develVolumes=" ".join(
-          '-v "$PWD/$(readlink {pkg} || echo {pkg})":/{pkg}:rw'.format(pkg=quote(spec["package"]))
-          for spec in specs.values() if spec["is_devel_pkg"]),
-        additionalVolumes=" ".join(
-          "-v %s" % quote(volume) for volume in args.volumes),
-        mirrorVolume=("-v %s:/mirror" % quote(dirname(spec["reference"]))
-                      if "reference" in spec else ""),
-      )
+      build_command = container_run_string(args.dockerImage, args.workDir, args.configDir, scriptDir,
+                                           args.docker_extra_args, spec, specs, args.volumes, buildEnvironment)
+
     else:
       os.environ.update(buildEnvironment)
       build_command = f"{BASH} -e -x {quote(scriptDir)}/build.sh 2>&1"

alibuild_helpers/build_template.sh

@@ -293,7 +293,7 @@ mkdir -p "${WORK_DIR}/TARS/$HASH_PATH" \
 PACKAGE_WITH_REV=$PKGNAME-$PKGVERSION-$PKGREVISION.$ARCHITECTURE.tar.gz
 # Copy and tar/compress (if applicable) in parallel.
 # Use -H to match tar's behaviour of preserving hardlinks.
-rsync -aH "$WORK_DIR/INSTALLROOT/$PKGHASH/" "$WORK_DIR" & rsync_pid=$!
+rsync -DgloprH "$WORK_DIR/INSTALLROOT/$PKGHASH/" "$WORK_DIR" & rsync_pid=$!
 if [ "$CAN_DELETE" = 1 ]; then
   # We're deleting the tarball anyway, so no point in creating a new one.
   # There might be an old existing tarball, and we should delete it.

alibuild_helpers/cmd.py

@@ -5,6 +5,7 @@
 from textwrap import dedent
 from subprocess import TimeoutExpired
 from shlex import quote
+import platform
 
 from alibuild_helpers.log import debug, error, dieOnError
 
@@ -70,6 +71,52 @@ def execute(command, printer=debug, timeout=None):
 
 BASH = "bash" if getstatusoutput("/bin/bash --version")[0] else "/bin/bash"
 
+class AppleContainerRunner:
+    """A context manager for running commands inside a Apple Container (see https://github.com/apple/container)
+       If the image given is None or empty, the commands are run on the host instead.
+    """
+    def __init__(self, container_image, container_run_args=(), extra_env={}, extra_volumes=[]) -> None:
+        self._container_image = container_image
+        self._container_run_args = container_run_args
+        self._container = None
+        self._extra_env = extra_env
+        self._extra_volumes = extra_volumes
+
+    def __enter__(self):
+        def getstatusoutput_host(cmd, cwd=None):
+            command_prefix=""
+            if self._extra_env:
+                command_prefix="env " + " ".join("{}={}".format(k, quote(v)) for (k,v) in self._extra_env.items()) + " "
+            return getstatusoutput("{}{} -c {}".format(command_prefix, BASH, quote(cmd))
+                                , cwd=cwd)
+
+        if not self._container_image:
+            return getstatusoutput_host
+
+        envOpts = [opt for k, v in self._extra_env.items() for opt in ("-e", f"{k}={v}")]
+        volumes = [opt for v in self._extra_volumes for opt in ("-v", v)]
+        # Apple Container is more picky about missing entrypoints, so we always override it
+        # with /bin/sleep.
+        cmd = ["container", "run", "--detach"] + envOpts + volumes + ["--rm", "--entrypoint=/bin/sleep"]
+        cmd += self._container_run_args
+        cmd += [self._container_image, "inf"]
+        self._container = getoutput(cmd).strip()
+
+        def getstatusoutput_container(cmd, cwd=None):
+            if self._container is None:
+                return getstatusoutput_host(cmd, cwd=cwd)
+            envOpts = [opt for k, v in self._extra_env.items() for opt in ("-e", f"{k}={v}")]
+            exec_cmd = ["container", "exec"] + envOpts + [self._container, "bash", "-c", cmd]
+            return getstatusoutput(exec_cmd, cwd=cwd)
+
+        return getstatusoutput_container
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self._container is not None:
+            getstatusoutput("container kill " + quote(self._container))
+        self._container = None
+        return False  # propagate any exception that may have occurred
+
 
 class DockerRunner:
   """A context manager for running commands inside a Docker container.
@@ -146,3 +193,84 @@ def install_wrapper_script(name, work_dir):
              "rerunning this command inside a login shell (e.g. `bash -l`). "
              "If that doesn't work, run `export PATH` manually.")
   os.environ["PATH"] = script_dir + ":" + os.environ["PATH"]
+
+def to_int(s: str) -> int:
+    try:
+        return int(float(s))
+    except (ValueError, TypeError):
+        return 0
+
+
+def _apple_run_string(dockerImage, workDir, configDir, scriptDir, docker_extra_args, spec, specs, volumes, buildEnvironment):
+    build_command = (
+        "container run --rm --entrypoint=/bin/bash --user $(id -u):$(id -g) "
+        "--mount type=bind,source={workdir},target=/sw "
+        "--mount type=bind,source={configDir},target=/alidist,readonly "
+        "--mount type=bind,source={scriptDir},target=/scripts,readonly "
+        "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} "
+        "-e WORK_DIR_OVERRIDE=/sw -e ALIBUILD_CONFIG_DIR_OVERRIDE=/alidist {extraArgs} {image} -ex /scripts/build.sh"
+    ).format(
+        image=quote(dockerImage),
+        workdir=quote(os.path.abspath(workDir)),
+        configDir=quote(os.path.abspath(configDir)),
+        scriptDir=quote(scriptDir),
+        extraArgs=" ".join(map(quote, docker_extra_args)),
+        additionalEnv=" ".join(
+            "-e {}={}".format(var, quote(value)) for var, value in buildEnvironment
+        ),
+        # Used e.g. by O2DPG-sim-tests to find the O2DPG repository.
+        develVolumes=" ".join(
+            '--mount type=bind,source="$PWD/$(readlink {pkg} || echo {pkg})",target=/{pkg},readonly'.format(
+                pkg=quote(s["package"])
+            )
+            for s in specs.values()
+            if s["is_devel_pkg"]
+        ),
+        additionalVolumes=" ".join("--mount type=bind,source=%s" % quote(volume) for volume in volumes),
+        mirrorVolume=(
+            "--mount type=bind,source=%s,target=/mirror" % quote(os.path.dirname(spec["reference"]))
+            if "reference" in spec
+            else ""
+        ),
+    )
+    return build_command
+
+def _docker_run_string( dockerImage, workDir, configDir, scriptDir, docker_extra_args, spec, specs, volumes, buildEnvironment):
+    build_command = (
+        "docker run --rm --entrypoint= --user $(id -u):$(id -g) "
+        "-v {workdir}:/sw -v{configDir}:/alidist:ro -v {scriptDir}/build.sh:/build.sh:ro "
+        "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} "
+        "-e WORK_DIR_OVERRIDE=/sw -e ALIBUILD_CONFIG_DIR_OVERRIDE=/alidist {extraArgs} --network=host {image} bash -ex /build.sh"
+    ).format(
+        image=quote(dockerImage),
+        workdir=quote(os.path.abspath(workDir)),
+        configDir=quote(os.path.abspath(configDir)),
+        scriptDir=quote(scriptDir),
+        extraArgs=" ".join(map(quote, docker_extra_args)),
+        additionalEnv=" ".join(
+            "-e {}={}".format(var, quote(value)) for var, value in buildEnvironment
+        ),
+        # Used e.g. by O2DPG-sim-tests to find the O2DPG repository.
+        develVolumes=" ".join(
+            '-v "$PWD/$(readlink {pkg} || echo {pkg})":/{pkg}:rw'.format(
+                pkg=quote(spec["package"])
+            )
+            for spec in specs.values()
+            if spec["is_devel_pkg"]
+        ),
+        additionalVolumes=" ".join("-v %s" % quote(volume) for volume in volumes),
+        mirrorVolume=(
+            "-v %s:/mirror" % quote(os.path.dirname(spec["reference"]))
+            if "reference" in spec
+            else ""
+        ),
+    )
+    return build_command
+
+
+if to_int(platform.mac_ver()[0].split(".")[0]) >= 26:
+    ContainerRunner = AppleContainerRunner
+    container_run_string = _apple_run_string
+else:
+    ContainerRunner = DockerRunner
+    container_run_string = _docker_run_string

alibuild_helpers/deps.py

@@ -2,7 +2,7 @@
 
 from alibuild_helpers.log import debug, dieOnError
 from alibuild_helpers.utilities import parseDefaults, readDefaults, getPackageList, validateDefaults
-from alibuild_helpers.cmd import DockerRunner, execute
+from alibuild_helpers.cmd import ContainerRunner, execute
 from os import path
 import sys
 
@@ -15,7 +15,7 @@ def doDeps(args, parser):
   extra_env = {"ALIBUILD_CONFIG_DIR": "/alidist" if args.docker else path.abspath(args.configDir)}
   extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
 
-  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
+  with ContainerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
     def performCheck(pkg, cmd):
       return getstatusoutput_docker(cmd)
 

alibuild_helpers/doctor.py

@@ -7,7 +7,7 @@
 from alibuild_helpers.log import debug, error, banner, info, success, warning
 from alibuild_helpers.log import logger
 from alibuild_helpers.utilities import getPackageList, parseDefaults, readDefaults, validateDefaults
-from alibuild_helpers.cmd import getstatusoutput, DockerRunner
+from alibuild_helpers.cmd import getstatusoutput, ContainerRunner
 import tempfile
 
 def prunePaths(workDir) -> None:
@@ -89,7 +89,7 @@ def doDoctor(args, parser):
   extra_env = {"ALIBUILD_CONFIG_DIR": "/alidist" if args.docker else os.path.abspath(args.configDir)}
   extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
 
-  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
+  with ContainerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
     err, output = getstatusoutput_docker("type c++")
   if err:
     warning("Unable to find system compiler.\n"
@@ -149,7 +149,7 @@ def performValidateDefaults(spec):
   extra_env = {"ALIBUILD_CONFIG_DIR": "/alidist" if args.docker else os.path.abspath(args.configDir)}
   extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
 
-  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
+  with ContainerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
     fromSystem, own, failed, validDefaults = \
       getPackageList(packages                = packages,
                      specs                   = specs,

tests/test_args.py

@@ -52,10 +52,10 @@ class FakeExit(Exception):
   ((), "build --force-unknown-architecture zlib --no-remote-store --remote-store rsync://test.local/", [("noSystem", None), ("remoteStore", "")]),
   ((), "build zlib --architecture slc7_x86-64"                                         , [("noSystem", "*"), ("preferSystem", False), ("remoteStore", "https://s3.cern.ch/swift/v1/alibuild-repo")]),
   ((), "build zlib --architecture ubuntu1804_x86-64"                                   , [("noSystem", None), ("preferSystem", False), ("remoteStore", "")]),
-  ((), "build zlib -a slc7_x86-64"                                                     , [("docker", False), ("dockerImage", None), ("docker_extra_args", ["--network=host"])]),
+  ((), "build zlib -a slc7_x86-64"                                                     , [("docker", False), ("dockerImage", None), ("docker_extra_args", [])]),
   ((), "build zlib -a slc7_x86-64 --docker-image registry.cern.ch/alisw/some-builder"  , [("docker", True), ("dockerImage", "registry.cern.ch/alisw/some-builder")]),
   ((), "build zlib -a slc7_x86-64 --docker"                                            , [("docker", True), ("dockerImage", "registry.cern.ch/alisw/slc7-builder")]),
-  ((), "build zlib -a slc7_x86-64 --docker-extra-args=--foo"                           , [("docker", True), ("dockerImage", "registry.cern.ch/alisw/slc7-builder"), ("docker_extra_args", ["--foo", "--network=host"])]),
+  ((), "build zlib -a slc7_x86-64 --docker-extra-args=--foo"                           , [("docker", True), ("dockerImage", "registry.cern.ch/alisw/slc7-builder"), ("docker_extra_args", ["--foo"])]),
   ((), "build zlib --devel-prefix -a slc7_x86-64 --docker"                             , [("docker", True), ("dockerImage", "registry.cern.ch/alisw/slc7-builder"), ("develPrefix", "%s-slc7_x86-64" % os.path.basename(os.getcwd()))]),
   ((), "build zlib --devel-prefix -a slc7_x86-64 --docker-image someimage"             , [("docker", True), ("dockerImage", "someimage"), ("develPrefix", "%s-slc7_x86-64" % os.path.basename(os.getcwd()))]),
   ((), "--debug build --force-unknown-architecture --defaults o2 O2"                   , [("debug", True), ("action",  "build"), ("defaults", "o2"), ("pkgname", ["O2"])]),