diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 9b422cc1c5..306e543fe8 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -100,10 +100,7 @@ jobs: defaults: run: working-directory: runner - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Go @@ -126,11 +123,6 @@ jobs: if [[ "${{ inputs.go-integration-tests }}" == "true" ]]; then SHORT="" fi - # Skip failing integration tests on macOS for release builds. - # TODO: https://github.com/dstackai/dstack/issues/3005 - if [[ "${{ inputs.staging }}" == "false" && "${{ startsWith(matrix.os, 'macos') }}" == "true" ]]; then - SHORT="-short" - fi go version go fmt $(go list ./... | grep -v /vendor/) go vet $(go list ./... | grep -v /vendor/) diff --git a/mkdocs/docs/concepts/fleets.md b/mkdocs/docs/concepts/fleets.md index 85cb15b7e7..497ff468cb 100644 --- a/mkdocs/docs/concepts/fleets.md +++ b/mkdocs/docs/concepts/fleets.md @@ -103,7 +103,7 @@ To create a fleet, define its configuration in a YAML file. The filename must en `dstack apply` automatically connects to on-prem servers, installs the required dependencies, and adds them to the created fleet. ??? info "Host requirements" - 1. Hosts must be pre-installed with Docker. + 1. Hosts must be Linux-based and have Docker pre-installed. === "NVIDIA" 2. Hosts with NVIDIA GPUs must also be pre-installed with CUDA 12.1 and diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index 086ea80ad8..6ca27ac816 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -157,7 +157,6 @@ For more details on the options below, refer to the [server deployment](../guide * `DSTACK_SHIM_DOWNLOAD_URL` – Overrides `dstack-shim` binary download URL. The URL can contain `{version}` and/or `{arch}` placeholders, see `DSTACK_RUNNER_DOWNLOAD_URL` for the details. * `DSTACK_DEFAULT_CREDS_DISABLED` – Disables default credentials detection if set. Defaults to `None`. - * `DSTACK_LOCAL_BACKEND_ENABLED` – Enables local backend for debug if set. Defaults to `None`. ## CLI diff --git a/runner/.justfile b/runner/.justfile index 2dd452de2a..5419f9eb4d 100644 --- a/runner/.justfile +++ b/runner/.justfile @@ -39,6 +39,9 @@ export shim_download_url := "s3://" + s3_bucket + "/" + version + "/binaries/dst export shim_os := "" export shim_arch := "" +# Go toolchain image for running tests in a container (keep in sync with go.mod) +export go_version := env("DSTACK_GO_VERSION", "1.25") + # Build runner [private] build-runner-binary: @@ -78,10 +81,23 @@ clean-runner: rm -f {{source_directory()}}/cmd/shim/shim echo "Build artifacts cleaned!" -# Run tests for runner and shim +# Run tests for runner and shim (native; requires a Linux host) test-runner: cd {{source_directory()}} && go test -v ./... +# Run tests for runner and shim in a Linux container (use on macOS/Windows, where native builds are not available) +# Examples: +# just test-runner-in-container # short suite, all packages +# just test-runner-in-container -run TestPullImage ./internal/shim/ +test-runner-in-container *args="-short ./...": + docker run --rm -t \ + -v {{source_directory()}}:/src -w /src \ + -v dstack-go-mod:/go/pkg/mod \ + -v dstack-go-build:/root/.cache/go-build \ + -v /var/run/docker.sock:/var/run/docker.sock \ + golang:{{go_version}} \ + go test -race {{args}} + # Validate shim is built for linux/amd64 [private] validate-shim-binary: diff --git a/runner/README.md b/runner/README.md index be8a4610e4..be91a9b178 100644 --- a/runner/README.md +++ b/runner/README.md @@ -2,9 +2,19 @@ For overview of `dstack-shim` and `dstack-runner`, see [/contributing/RUNNER-AND-SHIM.md](../contributing/RUNNER-AND-SHIM.md). -## Running locally +`dstack-shim` and `dstack-runner` can be built only for GOOS=linux. Use containers for development on other OS. -Here's the steps to build `dstack-shim` and `dstack-runner` and run `dstack` with them locally: +## Testing locally + +Run shim and runner tests on any OS inside a Docker container: + +```shell +just test-runner-in-container +``` + +## Running locally (standalone) + +Build `dstack-shim` and `dstack-runner` and run them locally: 1. Build the runner executable @@ -13,8 +23,6 @@ Here's the steps to build `dstack-shim` and `dstack-runner` and run `dstack` wit go build ``` - Note: The runner runs inside the Docker container, so ensure it's compiled for linux/amd64. For example, on macOS you'd run `GOOS=linux GOARCH=amd64 go build`. - 2. Build the shim executable ```shell @@ -40,52 +48,14 @@ Now you can call shim API: >>> s.submit("","", "ubuntu", None) ``` -### Local backend - -You can also run `dstack` end-to-end with local shim and runner by enabling the `local` backend on dstack server: - -```shell -DSTACK_LOCAL_BACKEND_ENABLED= dstack server --log-level=debug -``` - -The `local` backend will submit the run to the locally started shim and runner. The CLI will attach to the container just as if it were any other cloud backend: - -```shell -✗ dstack apply . - Configuration .dstack.yml - Project main - User admin - Pool name default-pool - Min resources 2..xCPU, 4GB.. - Max price - - Max duration 6h - Spot policy auto - Retry policy yes - Creation policy reuse-or-create - Termination policy destroy-after-idle - Termination idle time 300s - - # BACKEND REGION INSTANCE RESOURCES SPOT PRICE - 1 local local local 4xCPU, 8GB, 100GB no $0 - (disk) - 2 azure westeurope Standard_D2s_v3 2xCPU, 8GB, 100GB yes $0.012 - (disk) - 3 azure westeurope Standard_E2s_v4 2xCPU, 16GB, 100GB yes $0.015246 - (disk) - ... - Shown 3 of 4041 offers, $56.6266 max - -Continue? [y/n]: -``` - -## Testing remotely +## Running with `dstack` -You can also test the built shim and runner using standard backends (including SSH fleets). +You can test the built shim and runner with `dstack` using standard backends (including SSH fleets). > [!NOTE] -> To run with standard backends, both the runner and shim must be built for linux/amd64. +> To run with standard backends, both the runner and shim must be built for linux. -Build the runner and shim, and upload them to S3 automatically using `just` (see [`justfile`](justfile)). +Build the runner and shim and upload them to S3 using `just` (see [`justfile`](justfile)). > [!IMPORTANT] > Before running any `just` commands that upload to S3, you must set the following environment variables: @@ -101,7 +71,7 @@ Build the runner and shim, and upload them to S3 automatically using `just` (see just upload ``` -To use the built shim and runner with the dstack server, pass the URLs via `DSTACK_SHIM_DOWNLOAD_URL` and `DSTACK_RUNNER_DOWNLOAD_URL`: +To use the built shim and runner with the `dstack` server, pass the URLs via `DSTACK_SHIM_DOWNLOAD_URL` and `DSTACK_RUNNER_DOWNLOAD_URL`: ```shell export DSTACK_SHIM_DOWNLOAD_URL="https://${DSTACK_SHIM_UPLOAD_S3_BUCKET}.s3.amazonaws.com/${DSTACK_SHIM_UPLOAD_VERSION}/binaries/dstack-shim-linux-amd64" @@ -112,7 +82,7 @@ dstack server --log-level=debug ## Dependencies (WIP) -These are nonexhaustive lists of external dependencies (executables, libraries) of the `dstack-*` binaries. +These are non-exhaustive lists of external dependencies (executables, libraries) of the `dstack-*` binaries. **TODO**: inspect codebase, add missing dependencies. diff --git a/runner/cmd/runner/main.go b/runner/cmd/runner/main.go index 258523f5ae..08196b0ba4 100644 --- a/runner/cmd/runner/main.go +++ b/runner/cmd/runner/main.go @@ -1,3 +1,7 @@ +//go:build linux + +// dstack-runner is supported only in Linux environments. + package main import ( diff --git a/runner/cmd/shim/main.go b/runner/cmd/shim/main.go index 78cc405bf9..116e16b50c 100644 --- a/runner/cmd/shim/main.go +++ b/runner/cmd/shim/main.go @@ -1,3 +1,7 @@ +//go:build linux + +// dstack-shim is supported only in Linux environments. + package main import ( diff --git a/runner/internal/runner/executor/executor.go b/runner/internal/runner/executor/executor.go index 2c3f3f03f0..31f3d7fe92 100644 --- a/runner/internal/runner/executor/executor.go +++ b/runner/internal/runner/executor/executor.go @@ -11,7 +11,6 @@ import ( "os/exec" "path" "path/filepath" - "runtime" "strconv" "strings" "sync" @@ -90,34 +89,19 @@ type RunExecutor struct { connectionTracker ConnectionTracker } -// stubConnectionTracker is a no-op implementation for when procfs is not available (only required for tests on darwin) -type stubConnectionTracker struct{} - -func (s *stubConnectionTracker) GetNoConnectionsSecs() int64 { return 0 } -func (s *stubConnectionTracker) Track(ticker <-chan time.Time) {} -func (s *stubConnectionTracker) Stop() {} - func NewRunExecutor(tempDir string, dstackDir string, currentUser linuxuser.User, sshd ssh.SshdManager) (*RunExecutor, error) { mu := &sync.RWMutex{} timestamp := NewMonotonicTimestamp() - // Try to initialize procfs, but don't fail if it's not available (e.g., on macOS) - var connectionTracker ConnectionTracker - - if runtime.GOOS == "linux" { - proc, err := procfs.NewDefaultFS() - if err != nil { - return nil, fmt.Errorf("initialize procfs: %w", err) - } - connectionTracker = connections.NewConnectionTracker(connections.ConnectionTrackerConfig{ - Port: uint64(sshd.Port()), - MinConnDuration: 10 * time.Second, // shorter connections are likely from dstack-server - Procfs: proc, - }) - } else { - // Use stub connection tracker (only required for tests on darwin) - connectionTracker = &stubConnectionTracker{} + proc, err := procfs.NewDefaultFS() + if err != nil { + return nil, fmt.Errorf("initialize procfs: %w", err) } + connectionTracker := connections.NewConnectionTracker(connections.ConnectionTrackerConfig{ + Port: uint64(sshd.Port()), + MinConnDuration: 10 * time.Second, // shorter connections are likely from dstack-server + Procfs: proc, + }) return &RunExecutor{ tempDir: tempDir, diff --git a/runner/internal/runner/linux/capabilities/capabilities_linux.go b/runner/internal/runner/linux/capabilities/capabilities.go similarity index 98% rename from runner/internal/runner/linux/capabilities/capabilities_linux.go rename to runner/internal/runner/linux/capabilities/capabilities.go index c27e887a5d..fb70b60930 100644 --- a/runner/internal/runner/linux/capabilities/capabilities_linux.go +++ b/runner/internal/runner/linux/capabilities/capabilities.go @@ -1,5 +1,3 @@ -//go:build linux - package capabilities import ( diff --git a/runner/internal/runner/linux/capabilities/capabilities_darwin.go b/runner/internal/runner/linux/capabilities/capabilities_darwin.go deleted file mode 100644 index 6a60f94af1..0000000000 --- a/runner/internal/runner/linux/capabilities/capabilities_darwin.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build darwin - -package capabilities - -import "errors" - -type Capability string - -const ( - SETUID = Capability("SETUID") - SETGID = Capability("SETGID") - CHOWN = Capability("CHOWN") - SYS_RESOURCE = Capability("SYS_RESOURCE") -) - -func Has(c Capability) (bool, error) { - return false, errors.New("not supported") -} - -func Check(cs ...Capability) (missing []Capability, err error) { - return nil, errors.New("not supported") -} diff --git a/runner/internal/runner/metrics/metrics_test.go b/runner/internal/runner/metrics/metrics_test.go index 3410435ce2..844b02bd30 100644 --- a/runner/internal/runner/metrics/metrics_test.go +++ b/runner/internal/runner/metrics/metrics_test.go @@ -1,7 +1,6 @@ package metrics import ( - "runtime" "testing" "github.com/dstackai/dstack/runner/internal/runner/schemas" @@ -9,9 +8,6 @@ import ( ) func TestGetAMDGPUMetrics_OK(t *testing.T) { - if runtime.GOOS == "darwin" { - t.Skip("Skipping on macOS") - } collector, err := NewMetricsCollector(t.Context()) assert.NoError(t, err) @@ -43,9 +39,6 @@ func TestGetAMDGPUMetrics_OK(t *testing.T) { } func TestGetAMDGPUMetrics_ErrorGPUUtilNA(t *testing.T) { - if runtime.GOOS == "darwin" { - t.Skip("Skipping on macOS") - } collector, err := NewMetricsCollector(t.Context()) assert.NoError(t, err) metrics, err := collector.getAMDGPUMetrics("gpu,gfx,gfx_clock,vram_used,vram_total\n0,N/A,N/A,283,196300\n") diff --git a/runner/internal/shim/dcgm/wrapper_linux.go b/runner/internal/shim/dcgm/wrapper_cgo.go similarity index 99% rename from runner/internal/shim/dcgm/wrapper_linux.go rename to runner/internal/shim/dcgm/wrapper_cgo.go index 359d374ea0..779decea9b 100644 --- a/runner/internal/shim/dcgm/wrapper_linux.go +++ b/runner/internal/shim/dcgm/wrapper_cgo.go @@ -1,4 +1,4 @@ -//go:build linux && cgo +//go:build cgo package dcgm diff --git a/runner/internal/shim/dcgm/wrapper_linux_test.go b/runner/internal/shim/dcgm/wrapper_cgo_test.go similarity index 99% rename from runner/internal/shim/dcgm/wrapper_linux_test.go rename to runner/internal/shim/dcgm/wrapper_cgo_test.go index 20fc4d59a6..e596b4eeba 100644 --- a/runner/internal/shim/dcgm/wrapper_linux_test.go +++ b/runner/internal/shim/dcgm/wrapper_cgo_test.go @@ -1,4 +1,4 @@ -//go:build linux && cgo +//go:build cgo package dcgm diff --git a/runner/internal/shim/dcgm/wrapper_darwin.go b/runner/internal/shim/dcgm/wrapper_darwin.go deleted file mode 100644 index 590e9e9bd0..0000000000 --- a/runner/internal/shim/dcgm/wrapper_darwin.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build darwin - -package dcgm - -import "errors" - -func NewDCGMWrapper(address string) (DCGMWrapperInterface, error) { - return nil, errors.New("macOS is not supported") -} diff --git a/runner/internal/shim/dcgm/wrapper_linux_nocgo.go b/runner/internal/shim/dcgm/wrapper_nocgo.go similarity index 88% rename from runner/internal/shim/dcgm/wrapper_linux_nocgo.go rename to runner/internal/shim/dcgm/wrapper_nocgo.go index 60c97e49e1..730c376e8c 100644 --- a/runner/internal/shim/dcgm/wrapper_linux_nocgo.go +++ b/runner/internal/shim/dcgm/wrapper_nocgo.go @@ -1,4 +1,4 @@ -//go:build linux && !cgo +//go:build !cgo package dcgm diff --git a/runner/internal/shim/docker.go b/runner/internal/shim/docker.go index 604cc83548..a1318e4243 100644 --- a/runner/internal/shim/docker.go +++ b/runner/internal/shim/docker.go @@ -13,7 +13,6 @@ import ( "os/exec" "os/user" "path/filepath" - rt "runtime" "strconv" "strings" "sync" @@ -966,9 +965,6 @@ func (d *DockerRunner) startContainer(ctx context.Context, task *Task) error { if err != nil { return fmt.Errorf("inspect container: %w", err) } - // FIXME: container_.NetworkSettings.Ports values (bindings) are not immediately available - // on macOS, so ports can be empty with local backend. - // Workaround: restart shim after submitting the run. task.ports = extractPorts(ctx, container_.NetworkSettings.Ports) return nil } @@ -1083,10 +1079,7 @@ func extractPorts(ctx context.Context, portMap nat.PortMap) []PortMapping { } func getNetworkMode(networkMode NetworkMode) container.NetworkMode { - if rt.GOOS == "linux" { - return container.NetworkMode(networkMode) - } - return "default" + return container.NetworkMode(networkMode) } func configureGpuDevices(hostConfig *container.HostConfig, gpuDevices []GPUDevice) { diff --git a/runner/internal/shim/docker_test.go b/runner/internal/shim/docker_test.go index e8dbdc3508..41627da20a 100644 --- a/runner/internal/shim/docker_test.go +++ b/runner/internal/shim/docker_test.go @@ -4,8 +4,6 @@ import ( "context" "encoding/hex" "math/rand" - "os" - "runtime" "sync" "testing" "time" @@ -18,7 +16,7 @@ import ( // TestDocker_SSHServer pulls ubuntu image (without sshd), installs openssh-server and exits // Basically, it indirectly tests a shell script generated by getSSHShellCommands func TestDocker_SSHServer(t *testing.T) { - if testing.Short() || (os.Getenv("CI") == "true" && runtime.GOOS == "darwin") { + if testing.Short() { t.Skip() } t.Parallel() @@ -44,7 +42,7 @@ func TestDocker_SSHServer(t *testing.T) { } func TestDocker_ShmNoexecByDefault(t *testing.T) { - if testing.Short() || (os.Getenv("CI") == "true" && runtime.GOOS == "darwin") { + if testing.Short() { t.Skip() } t.Parallel() @@ -69,7 +67,7 @@ func TestDocker_ShmNoexecByDefault(t *testing.T) { } func TestDocker_ShmExecIfSizeSpecified(t *testing.T) { - if testing.Short() || (os.Getenv("CI") == "true" && runtime.GOOS == "darwin") { + if testing.Short() { t.Skip() } t.Parallel() diff --git a/src/dstack/_internal/core/backends/features.py b/src/dstack/_internal/core/backends/features.py index 811c90b0d5..d5c28728dc 100644 --- a/src/dstack/_internal/core/backends/features.py +++ b/src/dstack/_internal/core/backends/features.py @@ -12,9 +12,7 @@ ) from dstack._internal.core.backends.base.configurator import Configurator from dstack._internal.core.backends.configurators import list_available_configurator_classes -from dstack._internal.core.backends.local.compute import LocalCompute from dstack._internal.core.models.backends.base import BackendType -from dstack._internal.settings import LOCAL_BACKEND_ENABLED _configurator_classes = list_available_configurator_classes() @@ -27,8 +25,6 @@ def _get_backends_with_compute_feature( (configurator_class.TYPE, configurator_class.BACKEND_CLASS.COMPUTE_CLASS) for configurator_class in configurator_classes ] - if LOCAL_BACKEND_ENABLED: - backend_types_and_computes.append((BackendType.LOCAL, LocalCompute)) backend_types = [] for backend_type, compute_class in backend_types_and_computes: if issubclass(compute_class, compute_feature_class): diff --git a/src/dstack/_internal/core/backends/local/__init__.py b/src/dstack/_internal/core/backends/local/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/dstack/_internal/core/backends/local/backend.py b/src/dstack/_internal/core/backends/local/backend.py deleted file mode 100644 index b279bc7b5f..0000000000 --- a/src/dstack/_internal/core/backends/local/backend.py +++ /dev/null @@ -1,14 +0,0 @@ -from dstack._internal.core.backends.base.backend import Backend -from dstack._internal.core.backends.local.compute import LocalCompute -from dstack._internal.core.models.backends.base import BackendType - - -class LocalBackend(Backend): - TYPE = BackendType.LOCAL - COMPUTE_CLASS = LocalCompute - - def __init__(self): - self._compute = LocalCompute() - - def compute(self) -> LocalCompute: - return self._compute diff --git a/src/dstack/_internal/core/backends/local/compute.py b/src/dstack/_internal/core/backends/local/compute.py deleted file mode 100644 index 03a489acaf..0000000000 --- a/src/dstack/_internal/core/backends/local/compute.py +++ /dev/null @@ -1,132 +0,0 @@ -from collections.abc import Iterator -from typing import List, Optional - -from dstack._internal.core.backends.base.compute import ( - Compute, - ComputeWithCreateInstanceSupport, - ComputeWithInstanceVolumesSupport, - ComputeWithPrivilegedSupport, - ComputeWithVolumeSupport, -) -from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT -from dstack._internal.core.models.backends.base import BackendType -from dstack._internal.core.models.instances import ( - InstanceAvailability, - InstanceConfiguration, - InstanceOfferWithAvailability, - InstanceRuntime, - InstanceType, - Resources, -) -from dstack._internal.core.models.placement import PlacementGroup -from dstack._internal.core.models.runs import Job, JobProvisioningData, Requirements, Run -from dstack._internal.core.models.volumes import ( - Volume, - VolumeAttachmentData, - VolumeProvisioningData, -) -from dstack._internal.utils.common import get_or_error -from dstack._internal.utils.logging import get_logger - -logger = get_logger(__name__) - - -class LocalCompute( - ComputeWithCreateInstanceSupport, - ComputeWithPrivilegedSupport, - ComputeWithInstanceVolumesSupport, - ComputeWithVolumeSupport, - Compute, -): - def get_offers(self, requirements: Requirements) -> Iterator[InstanceOfferWithAvailability]: - return iter( - [ - InstanceOfferWithAvailability( - backend=BackendType.LOCAL, - instance=InstanceType( - name="local", - resources=Resources(cpus=4, memory_mib=8192, gpus=[], spot=False), - ), - region="local", - price=0.00, - availability=InstanceAvailability.AVAILABLE, - instance_runtime=InstanceRuntime.RUNNER, - ) - ] - ) - - def terminate_instance( - self, instance_id: str, region: str, backend_data: Optional[str] = None - ): - pass - - def create_instance( - self, - instance_offer: InstanceOfferWithAvailability, - instance_config: InstanceConfiguration, - placement_group: Optional[PlacementGroup], - ) -> JobProvisioningData: - return JobProvisioningData( - backend=instance_offer.backend, - instance_type=instance_offer.instance, - instance_id="local", - hostname="127.0.0.1", - internal_ip=None, - region="", - price=instance_offer.price, - username="root", - ssh_port=DSTACK_RUNNER_SSH_PORT, - ssh_proxy=None, - dockerized=True, - backend_data=None, - ) - - def run_job( - self, - run: Run, - job: Job, - instance_offer: InstanceOfferWithAvailability, - project_ssh_public_key: str, - project_ssh_private_key: str, - volumes: List[Volume], - placement_group: Optional[PlacementGroup], - ) -> JobProvisioningData: - return JobProvisioningData( - backend=instance_offer.backend, - instance_type=instance_offer.instance, - instance_id="local", - hostname="127.0.0.1", - internal_ip=None, - region="", - price=instance_offer.price, - username="root", - ssh_port=DSTACK_RUNNER_SSH_PORT, - ssh_proxy=None, - dockerized=True, - backend_data=None, - ) - - def register_volume(self, volume: Volume) -> VolumeProvisioningData: - return VolumeProvisioningData( - volume_id=get_or_error(volume.volume_id), - size_gb=volume.configuration.size_gb, - ) - - def create_volume(self, volume: Volume) -> VolumeProvisioningData: - return VolumeProvisioningData( - volume_id=volume.name, - size_gb=volume.configuration.size_gb, - ) - - def delete_volume(self, volume: Volume): - pass - - def attach_volume( - self, volume: Volume, provisioning_data: JobProvisioningData - ) -> VolumeAttachmentData: - return VolumeAttachmentData(device_name=None) - - def detach_volume( - self, volume: Volume, provisioning_data: JobProvisioningData, force: bool = False - ): - pass diff --git a/src/dstack/_internal/core/models/backends/base.py b/src/dstack/_internal/core/models/backends/base.py index dd11ae67a3..80e241b315 100644 --- a/src/dstack/_internal/core/models/backends/base.py +++ b/src/dstack/_internal/core/models/backends/base.py @@ -42,7 +42,6 @@ class BackendType(str, enum.Enum): JARVISLABS = "jarvislabs" KUBERNETES = "kubernetes" LAMBDA = "lambda" - LOCAL = "local" REMOTE = "remote" NEBIUS = "nebius" OCI = "oci" diff --git a/src/dstack/_internal/core/services/ssh/attach.py b/src/dstack/_internal/core/services/ssh/attach.py index 2400113f2f..7727aef5df 100644 --- a/src/dstack/_internal/core/services/ssh/attach.py +++ b/src/dstack/_internal/core/services/ssh/attach.py @@ -169,7 +169,6 @@ def __init__( container_user: str, dockerized: bool, ssh_proxy: Optional[SSHConnectionParams] = None, - local_backend: bool = False, service_port: Optional[int] = None, bind_address: Optional[str] = None, ): @@ -182,17 +181,7 @@ def __init__( bind_address=bind_address, ) hosts = self.hosts - if local_backend: - hosts[run_name] = { - "HostName": hostname, - "Port": container_ssh_port, - "User": container_user, - "IdentityFile": self.identity_file, - "IdentitiesOnly": "yes", - "StrictHostKeyChecking": "no", - "UserKnownHostsFile": "/dev/null", - } - elif dockerized: + if dockerized: if ssh_proxy is not None: # SSH instance with jump host # dstack has no IdentityFile for jump host, it must be either preconfigured diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py index 98e5967cb8..014f84c604 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py @@ -13,7 +13,6 @@ from dstack._internal.core.consts import DSTACK_RUNNER_HTTP_PORT, DSTACK_SHIM_HTTP_PORT from dstack._internal.core.errors import GatewayError, SSHError -from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import NetworkMode, RegistryAuth from dstack._internal.core.models.configurations import ( DevEnvironmentConfiguration, @@ -749,8 +748,6 @@ async def _process_provisioning_status( assert context.run.run_spec.ssh_key_pub is not None user_ssh_key = context.run.run_spec.ssh_key_pub.strip() public_keys.append(user_ssh_key) - if job_provisioning_data.backend == BackendType.LOCAL: - user_ssh_key = None success = await run_async( _process_provisioning_with_shim, server_ssh_private_keys, diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py index d3cbcaebaa..f118261796 100644 --- a/src/dstack/_internal/server/services/backends/__init__.py +++ b/src/dstack/_internal/server/services/backends/__init__.py @@ -20,7 +20,6 @@ get_configurator, list_available_backend_types, ) -from dstack._internal.core.backends.local.backend import LocalBackend from dstack._internal.core.backends.models import ( AnyBackendConfigWithCreds, AnyBackendConfigWithoutCreds, @@ -43,7 +42,6 @@ from dstack._internal.server import settings from dstack._internal.server.models import BackendModel, DecryptedString, ProjectModel from dstack._internal.server.services.offers import merge_offer_iterables -from dstack._internal.settings import LOCAL_BACKEND_ENABLED from dstack._internal.utils.common import run_async from dstack._internal.utils.logging import get_logger @@ -400,8 +398,6 @@ async def get_project_backend_with_model_by_type_or_error( async def get_project_backends(project: ProjectModel) -> List[Backend]: backends_with_models = await get_project_backends_with_models(project) backends = [b for _, b in backends_with_models] - if LOCAL_BACKEND_ENABLED: - backends.append(LocalBackend()) return backends diff --git a/src/dstack/_internal/server/services/proxy/repo.py b/src/dstack/_internal/server/services/proxy/repo.py index e1778cfabb..e2a8d3c117 100644 --- a/src/dstack/_internal/server/services/proxy/repo.py +++ b/src/dstack/_internal/server/services/proxy/repo.py @@ -7,7 +7,6 @@ import dstack._internal.server.services.jobs as jobs_services from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT -from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.core.models.instances import SSHConnectionParams from dstack._internal.core.models.runs import ( @@ -105,9 +104,6 @@ async def get_service(self, project_name: str, run_name: str) -> Optional[Servic ssh_proxy_private_key = None if job.project_id != instance.project_id: ssh_proxy_private_key = instance.project.ssh_private_key - if jpd.backend == BackendType.LOCAL: - ssh_proxy = None - ssh_proxy_private_key = None ssh_head_proxy: Optional[SSHConnectionParams] = None ssh_head_proxy_private_key: Optional[str] = None rci = get_instance_remote_connection_info(instance) diff --git a/src/dstack/_internal/server/services/runner/ssh.py b/src/dstack/_internal/server/services/runner/ssh.py index b1430fba6f..19dfb05a7b 100644 --- a/src/dstack/_internal/server/services/runner/ssh.py +++ b/src/dstack/_internal/server/services/runner/ssh.py @@ -6,7 +6,6 @@ from typing_extensions import Concatenate, ParamSpec from dstack._internal.core.errors import DstackError, SSHError -from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.runs import JobProvisioningData, JobRuntimeData from dstack._internal.server import settings from dstack._internal.server.services.runner.client import LocalAddress @@ -53,13 +52,6 @@ def wrapper( Returns: is successful """ - if job_provisioning_data.backend == BackendType.LOCAL: - # without SSH - port_map = InstanceConnection.get_container_to_host_port_map( - job_provisioning_data, job_runtime_data - ) - return func(port_map, *args, **kwargs) - if not settings.SERVER_SSH_POOL_ENABLED or not job_provisioning_data.dockerized: # Connections from dstack-server to runner's sshd are expected to be short # as the `inactivity_duration` feature distinguishes user and server connections based on duration. diff --git a/src/dstack/_internal/server/services/ssh.py b/src/dstack/_internal/server/services/ssh.py index 72a0f65905..9d07263885 100644 --- a/src/dstack/_internal/server/services/ssh.py +++ b/src/dstack/_internal/server/services/ssh.py @@ -1,7 +1,6 @@ from collections.abc import Iterable from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT -from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.instances import SSHConnectionParams from dstack._internal.core.services.ssh.tunnel import SSH_DEFAULT_OPTIONS, SocketPair, SSHTunnel from dstack._internal.server.models import JobModel @@ -44,14 +43,13 @@ def get_container_ssh_credentials(job: JobModel) -> list[tuple[SSHConnectionPara job_project_key = FileContent(job.project.ssh_private_key) if jpd.dockerized: - if jpd.backend != BackendType.LOCAL: - instance_proxy = SSHConnectionParams( - hostname=jpd.hostname, - username=jpd.username, - port=jpd.ssh_port, - ) - instance_project_key = FileContent(instance.project.ssh_private_key) - hosts.append((instance_proxy, instance_project_key)) + instance_proxy = SSHConnectionParams( + hostname=jpd.hostname, + username=jpd.username, + port=jpd.ssh_port, + ) + instance_project_key = FileContent(instance.project.ssh_private_key) + hosts.append((instance_proxy, instance_project_key)) ssh_port = DSTACK_RUNNER_SSH_PORT jrd = get_job_runtime_data(job) if jrd is not None and jrd.ports is not None: diff --git a/src/dstack/_internal/settings.py b/src/dstack/_internal/settings.py index 18926632bf..2847fb84e3 100644 --- a/src/dstack/_internal/settings.py +++ b/src/dstack/_internal/settings.py @@ -32,10 +32,6 @@ # Can be used to disable control characters (e.g. for testing). CLI_RICH_FORCE_TERMINAL = environ.get_bool("DSTACK_CLI_RICH_FORCE_TERMINAL") -# Development settings - -LOCAL_BACKEND_ENABLED = os.getenv("DSTACK_LOCAL_BACKEND_ENABLED") is not None - class FeatureFlags: """ diff --git a/src/dstack/api/_public/runs.py b/src/dstack/api/_public/runs.py index ceb86faa44..1b3def5424 100644 --- a/src/dstack/api/_public/runs.py +++ b/src/dstack/api/_public/runs.py @@ -398,7 +398,6 @@ def attach( dockerized=provisioning_data.dockerized, ssh_proxy=provisioning_data.ssh_proxy, service_port=service_port, - local_backend=provisioning_data.backend == BackendType.LOCAL, bind_address=bind_address, ) diff --git a/src/tests/_internal/cli/services/configurators/test_profile.py b/src/tests/_internal/cli/services/configurators/test_profile.py index 405d56c18b..d3c363c0f2 100644 --- a/src/tests/_internal/cli/services/configurators/test_profile.py +++ b/src/tests/_internal/cli/services/configurators/test_profile.py @@ -34,8 +34,8 @@ def test_max_duration(self): def test_backends(self): profile = Profile(name="test") - modified, _ = apply_args(profile, ["-b", "local", "--backend", "aws"]) - profile.backends = ["local", "aws"] + modified, _ = apply_args(profile, ["-b", "gcp", "--backend", "aws"]) + profile.backends = ["gcp", "aws"] assert profile.dict() == modified.dict() def test_spot_policy_spot(self): diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index a46ec93f8d..b1a9942506 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -195,7 +195,7 @@ def get_dev_env_run_plan_dict( }, ], "files": [], - "backends": ["local", "aws", "azure", "gcp", "lambda", "runpod"], + "backends": ["aws", "azure", "gcp", "lambda", "runpod"], "regions": ["us"], "availability_zones": None, "instance_types": None, @@ -220,7 +220,7 @@ def get_dev_env_run_plan_dict( "configuration_path": "dstack.yaml", "file_archives": [], "profile": { - "backends": ["local", "aws", "azure", "gcp", "lambda", "runpod"], + "backends": ["aws", "azure", "gcp", "lambda", "runpod"], "regions": ["us"], "availability_zones": None, "instance_types": None, @@ -440,7 +440,7 @@ def get_dev_env_run_dict( }, ], "files": [], - "backends": ["local", "aws", "azure", "gcp", "lambda"], + "backends": ["aws", "azure", "gcp", "lambda"], "regions": ["us"], "availability_zones": None, "instance_types": None, @@ -465,7 +465,7 @@ def get_dev_env_run_dict( "configuration_path": "dstack.yaml", "file_archives": [], "profile": { - "backends": ["local", "aws", "azure", "gcp", "lambda"], + "backends": ["aws", "azure", "gcp", "lambda"], "regions": ["us"], "availability_zones": None, "instance_types": None, diff --git a/src/tests/_internal/server/services/backends/test_provisioning.py b/src/tests/_internal/server/services/backends/test_provisioning.py index 6c57c2c740..94e8c98be9 100644 --- a/src/tests/_internal/server/services/backends/test_provisioning.py +++ b/src/tests/_internal/server/services/backends/test_provisioning.py @@ -93,7 +93,7 @@ def test_patch_all_efa_instance_types(self, instance_type: str, suffix: str) -> @pytest.mark.parametrize("suffix", ["-base", "-devel"]) @pytest.mark.parametrize( "backend", - [BackendType.GCP, BackendType.AZURE, BackendType.LAMBDA, BackendType.LOCAL], + [BackendType.GCP, BackendType.AZURE, BackendType.LAMBDA], ) @pytest.mark.parametrize( "instance_type", diff --git a/src/tests/_internal/server/services/test_instances.py b/src/tests/_internal/server/services/test_instances.py index 1f5e1fb52b..aaa3e48d88 100644 --- a/src/tests/_internal/server/services/test_instances.py +++ b/src/tests/_internal/server/services/test_instances.py @@ -358,7 +358,7 @@ async def test_converts_instance(self, test_db, session: AsyncSession): expected_instance = Instance( id=instance_id, project_name=project.name, - backend=BackendType.LOCAL, + backend=BackendType.AWS, instance_type=InstanceType( name="instance", resources=Resources(cpus=1, memory_mib=512, spot=False, gpus=[]) ), @@ -383,8 +383,8 @@ async def test_converts_instance(self, test_db, session: AsyncSession): unreachable=False, health=HealthStatus.WARNING, project=project, - job_provisioning_data='{"ssh_proxy":null, "backend":"local","hostname":"hostname_test","region":"eu-west","price":1.0,"username":"user1","ssh_port":12345,"dockerized":false,"instance_id":"test_instance","instance_type": {"name": "instance", "resources": {"cpus": 1, "memory_mib": 512, "gpus": [], "spot": false, "disk": {"size_mib": 102400}, "description":""}}}', - offer='{"price":"LOCAL", "price":1.0, "backend":"local", "region":"eu-west-1", "availability":"available","instance": {"name": "instance", "resources": {"cpus": 1, "memory_mib": 512, "gpus": [], "spot": false, "disk": {"size_mib": 102400}, "description":""}}}', + job_provisioning_data='{"ssh_proxy":null, "backend":"aws","hostname":"hostname_test","region":"eu-west","price":1.0,"username":"user1","ssh_port":12345,"dockerized":false,"instance_id":"test_instance","instance_type": {"name": "instance", "resources": {"cpus": 1, "memory_mib": 512, "gpus": [], "spot": false, "disk": {"size_mib": 102400}, "description":""}}}', + offer='{"price":1.0, "backend":"aws", "region":"eu-west-1", "availability":"available","instance": {"name": "instance", "resources": {"cpus": 1, "memory_mib": 512, "gpus": [], "spot": false, "disk": {"size_mib": 102400}, "description":""}}}', total_blocks=1, busy_blocks=0, ) diff --git a/src/tests/_internal/server/services/test_ssh.py b/src/tests/_internal/server/services/test_ssh.py index d9c492e225..ec8fda453d 100644 --- a/src/tests/_internal/server/services/test_ssh.py +++ b/src/tests/_internal/server/services/test_ssh.py @@ -283,42 +283,3 @@ async def test_ssh_instance_with_head_proxy( FileContent(self.run_project_key), ), ] - - async def test_local_backend( - self, - session: AsyncSession, - instance_project: ProjectModel, - run: RunModel, - ): - instance = await create_instance( - session=session, project=instance_project, backend=BackendType.LOCAL - ) - jpd = get_job_provisioning_data( - backend=BackendType.LOCAL, - dockerized=True, - hostname="127.0.0.1", - username="root", - ssh_port=DSTACK_RUNNER_SSH_PORT, - ssh_proxy=None, - ) - job = await create_job( - session=session, - run=run, - instance=instance, - job_provisioning_data=jpd, - # jrd is tested in vm-based backend tests - job_runtime_data=None, - ) - - hosts = get_container_ssh_credentials(job) - - assert hosts == [ - ( - SSHConnectionParams( - hostname="localhost", - username="root", - port=DSTACK_RUNNER_SSH_PORT, - ), - FileContent(self.run_project_key), - ), - ]