Shell API and CLI: Add option for staging additional runtime targets

src/buildstream/_frontend/cli.py

@@ -20,6 +20,7 @@
 import shutil
 import click
 from .. import _yaml
+from .._frontend.app import App
 from .._exceptions import BstError, LoadError, AppError, RemoteError
 from .complete import main_bashcomplete, complete_path, CompleteUnhandled
 from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope
@@ -686,6 +687,13 @@ def show(app, elements, deps, except_, order, format_):
     metavar="HOSTPATH PATH",
     help="Mount a file or directory into the sandbox",
 )
+@click.option(
+    "--with",
+    "other_targets",
+    type=click.Path(readable=False),
+    multiple=True,
+    help="A additional target to stage into an element's sandbox environment",
+)
 @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox")
 @click.option(
     "--use-buildtree",
@@ -723,8 +731,9 @@ def show(app, elements, deps, except_, order, format_):
 @click.argument("command", type=click.STRING, nargs=-1)
 @click.pass_obj
 def shell(
-    app,
+    app: App,
     target,
+    other_targets,
     command,
     mount,
     isolate,
@@ -748,13 +757,38 @@ def shell(
     otherwise bst may respond to them instead. e.g.
 
     \b
-        bst shell example.bst -- df -h
+        bst shell base.bst -- df -h
 
     Use the --build option to create a temporary sysroot for
     building the element instead.
 
+    Use the --with option to stage the artifacts of other elements
+    into the temporary sysroot to make them available to run e.g.
+
+    \b
+        bst shell --with base.bst example.bst -- cat example.txt
+
     If no COMMAND is specified, the default is to attempt
     to run an interactive shell.
+
+    # Examples:
+
+    For all examples on this page:
+    - example.bst is a simple import element with no dependencies
+       that imports a file called example.txt
+    - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc).
+
+    \b
+       # Attempt to run an interactive shell with example.bst
+       bst shell example.bst
+       # Attempt to run an df -h with example.bst
+       bst shell example.bst -- df h
+       # In a workspace directory, attempt to shell into the workspace element
+       bst shell
+       # Attempt to run cat from base.bst to read example.txt from example.bst
+       bst shell --with base.bst example.bst -- cat example.txt
+       # Attempt to run an interactive shell with the sources and all dependencies of example.bst
+             bst shell --build example.bst
     """
 
     # Buildtree can only be used with build shells
@@ -776,6 +810,7 @@ def shell(
                 target,
                 scope,
                 app.shell_prompt,
+                other_targets=other_targets,
                 mounts=mounts,
                 isolate=isolate,
                 command=command,

src/buildstream/_stream.py

@@ -243,6 +243,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source
     #    target: The name of the element to run the shell for
     #    scope: The scope for the shell, only BUILD or RUN are valid (_Scope)
     #    prompt: A function to return the prompt to display in the shell
+    #    other_targets (Iterable[str]): The name of other elements to stage in the shell
     #    unique_id: (str): A unique_id to use to lookup an Element instance
     #    mounts: Additional directories to mount into the sandbox
     #    isolate (bool): Whether to isolate the environment like we do in builds
@@ -259,9 +260,10 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source
     def shell(
         self,
         target: str,
-        scope: int,
+        scope: _Scope,
         prompt: Callable[[Element], str],
         *,
+        other_targets: Iterable[str] = (),
         unique_id: Optional[str] = None,
         mounts: Optional[List[_HostMount]] = None,
         isolate: bool = False,
@@ -357,8 +359,40 @@ def shell(
             self._fetch([element])
             _pipeline.assert_sources_cached(self._context, [element])
 
+        # Load the other targets
+        try:
+            other_elements = self.load_selection(
+                    other_targets,
+                    selection=_PipelineSelection.RUN,
+                    load_artifacts=True,
+                    connect_artifact_cache=True,
+                    connect_source_cache=True,
+                    artifact_remotes=artifact_remotes,
+                    source_remotes=source_remotes,
+                    ignore_project_artifact_remotes=ignore_project_artifact_remotes,
+                    ignore_project_source_remotes=ignore_project_source_remotes,
+                )
+        except StreamError as e:
+            if e.reason == "deps-not-supported":
+                raise StreamError(
+                    "Only buildtrees are supported with artifact names",
+                    detail="Use the --build and --use-buildtree options to shell into a cached build tree",
+                    reason="only-buildtrees-supported",
+                ) from e
+            raise
+
+        self.query_cache(other_elements)
+        self._pull_missing_artifacts(other_elements)
+        missing_deps = [dep for dep in _pipeline.dependencies(other_elements, _Scope.RUN) if not dep._cached()]
+        if missing_deps:
+            raise StreamError(
+                "Elements need to be built or downloaded before staging a shell environment",
+                detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))),
+                reason="shell-missing-deps",
+            )
+
         return element._shell(
-            scope, mounts=mounts, isolate=isolate, prompt=prompt(element), command=command, usebuildtree=usebuildtree
+            scope, mounts=mounts, isolate=isolate, prompt=prompt(element), command=command, usebuildtree=usebuildtree, other_elements=other_elements
         )
 
     # build()

src/buildstream/element.py

@@ -72,7 +72,7 @@
 from itertools import chain
 import string
 from threading import Lock
-from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence
+from typing import Self, cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence, Tuple
 
 from pyroaring import BitMap  # pylint: disable=no-name-in-module
 
@@ -2058,9 +2058,19 @@ def _push(self):
     #    prompt (str): A suitable prompt string for PS1
     #    command (list): An argv to launch in the sandbox
     #    usebuildtree (bool): Use the buildtree as its source
+    #    other_elements (List[Element]): Optional list of other runtime elements to stage in the sandbox
     #
     # Returns: Exit code
-    def _shell(self, scope=None, *, mounts=None, isolate=False, prompt=None, command=None, usebuildtree=False):
+    def _shell(self,
+               scope:_Scope=None,
+               *,
+               mounts:List[Tuple[str,str]]=None,
+               isolate:bool=False,
+               prompt:str=None,
+               command:List[str]=None,
+               usebuildtree:bool=False,
+               other_elements:List[Self]=None
+               ):
 
         with self._prepare_sandbox(scope, shell=True, usebuildtree=usebuildtree) as sandbox:
             environment = sandbox._get_configured_environment() or self.get_environment()
@@ -2076,6 +2086,16 @@ def _shell(self, scope=None, *, mounts=None, isolate=False, prompt=None, command
             if prompt is not None:
                 environment["PS1"] = prompt
 
+            with self.timed_activity("Staging other_targets", silent_nested=True), self.__collect_overlaps(sandbox):
+                self.stage_dependency_artifacts(sandbox,other_elements)
+
+            # Stage artifacts from other_elements into the sandbox.
+            for element in other_elements:
+                # Stage deps in the sandbox root
+                with element.timed_activity("Integrating sandbox"), sandbox.batch():
+                    for dep in element._dependencies(_Scope.RUN):
+                        dep.integrate(sandbox)
+
             # Special configurations for non-isolated sandboxes
             if not isolate:
 

tests/integration/shell.py

@@ -46,18 +46,24 @@
 #    mount (tuple): A (host, target) tuple for the `--mount` option
 #    element (str): The element to build and run a shell with
 #    isolate (bool): Whether to pass --isolate to `bst shell`
+#    other_elements (list(str)): Other elements to stage in the sandbox
 #
-def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False):
+def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False,other_elements=[]):
     # Ensure the element is built
     result = cli.run_project_config(project=project, project_config=config, args=["build", element])
     assert result.exit_code == 0
+    for other_element in other_elements:
+        result = cli.run_project_config(project=project, project_config=config, args=["build", other_element])
+        assert result.exit_code == 0
 
     args = ["shell"]
     if isolate:
         args += ["--isolate"]
     if mount is not None:
         host_path, target_path = mount
         args += ["--mount", host_path, target_path]
+    for other_element in other_elements:
+        args += ["--with", other_element]
     args += [element, "--", *command]
 
     return cli.run_project_config(project=project, project_config=config, args=args)
@@ -85,6 +91,21 @@ def test_executable(cli, datafiles):
     assert result.exit_code == 0
     assert result.output == "Horseys!\n"
 
+# Test staging and running additional targets in the shell of the main target for debugging.
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+def test_with_other_targets(cli, datafiles):
+    project = str(datafiles)
+
+    # Show we can't cat in a shell for manual/import-file.bst
+    result = execute_shell(cli, project, ["/bin/cat", "test.txt"],element="manual/import-file.bst")
+    assert result.exit_code == 1, "Shouldn't be able to read content of test.txt as manual/import-file.bst is a simple import element with no dependencies"
+
+    # Show we can now cat with base.bst in a shell for manual/import-file.bst
+    result = execute_shell(cli, project, ["/bin/cat", "test.txt"],element="manual/import-file.bst",other_elements=["base.bst"])
+    assert result.exit_code == 0, "Should be able to read content of test.txt as we now stage in base.bst that provides /bin/cat"
+    assert result.output == "This is a test\n"
+
 
 # Test shell environment variable explicit assignments
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])