From 235c89b5daaacd838b8e8a77f2f87089e781beba Mon Sep 17 00:00:00 2001 From: Cheese Date: Wed, 19 Aug 2026 11:41:10 +0800 Subject: [PATCH 1/3] chore(integration): add E2B filesystem example --- .gitmodules | 3 +++ integration/tidbcloud-fs-e2b-example | 1 + 2 files changed, 4 insertions(+) create mode 160000 integration/tidbcloud-fs-e2b-example diff --git a/.gitmodules b/.gitmodules index 9653555..fabafdd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,6 @@ [submodule "ref/fs"] path = ref/fs url = git@github.com:tidbcloud/fs.git +[submodule "integration/tidbcloud-fs-e2b-example"] + path = integration/tidbcloud-fs-e2b-example + url = git@github.com:likidu/tidbcloud-fs-e2b-example.git diff --git a/integration/tidbcloud-fs-e2b-example b/integration/tidbcloud-fs-e2b-example new file mode 160000 index 0000000..9bac425 --- /dev/null +++ b/integration/tidbcloud-fs-e2b-example @@ -0,0 +1 @@ +Subproject commit 9bac425d7ff6730deb4f77a4978e686febc5705a From 721c5c898dfad7be8378952e8ca79c5a505e71bb Mon Sep 17 00:00:00 2001 From: Cheese Date: Wed, 19 Aug 2026 11:46:18 +0800 Subject: [PATCH 2/3] docs(spec): design filesystem layer fork workflows --- .../0032-filesystem-layer-fork-workflows.md | 289 ++++++++++++++++++ ...> 0033-homebrew-and-scoop-distribution.md} | 0 ...=> 0034-serverless-function-deployment.md} | 0 .../0012-install-and-update-distribution.md | 4 +- ref/drive9 | 2 +- 5 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 docs/spec/0032-filesystem-layer-fork-workflows.md rename docs/spec/{0032-homebrew-and-scoop-distribution.md => 0033-homebrew-and-scoop-distribution.md} (100%) rename docs/spec/{0033-serverless-function-deployment.md => 0034-serverless-function-deployment.md} (100%) diff --git a/docs/spec/0032-filesystem-layer-fork-workflows.md b/docs/spec/0032-filesystem-layer-fork-workflows.md new file mode 100644 index 0000000..f817b83 --- /dev/null +++ b/docs/spec/0032-filesystem-layer-fork-workflows.md @@ -0,0 +1,289 @@ +# Filesystem Layer Fork Workflows + +## Goal + +Complete the public TiDB Cloud Filesystem layer workflow exposed by Drive9 so users and agents can create one base overlay, fork zero-copy child timelines, inspect ancestry, mount writable or historical views, abandon rejected timelines, and commit only the selected result to the base file system. + +This spec extends the existing `ti fs` layer commands. It does not add a new top-level command, implement layer semantics inside `ti`, or call Drive9's internal APIs directly. + +## User Outcome + +After this spec, a user can seed a workspace once and fork independent child layers without copying the workspace: + +```bash +ti fs create-directory --path /research/q3-market +ti fs create-layer --base-root-path /research/q3-market --layer-name research-base --actor-id hermes --tag topic=q3-market +ti fs copy-file --from-local ./ --to-remote /research/q3-market/ --recursive --layer-id research-base +ti fs create-layer-checkpoint --layer-id research-base --checkpoint-id seed --label workspace-seed + +ti fs fork-layer --parent-layer-ref research-base --layer-name style-brief --actor-id hermes-brief --checkpoint-id seed +ti fs fork-layer --parent-layer-ref research-base --layer-name style-longform --actor-id hermes-longform --checkpoint-id seed +ti fs fork-layer --parent-layer-ref research-base --layer-name style-analyst --actor-id hermes-analyst --checkpoint-id seed + +ti fs list-layer-chain --layer-ref style-analyst +``` + +Each child is a copy-on-write timeline pinned to the selected parent tip or checkpoint. Forking does not copy the file tree. Changes remain outside the base file system until `commit-layer` succeeds. + +Users can mount one writable child and one historical checkpoint view: + +```bash +ti fs mount-file-system --mount-path ./runs/analyst --driver fuse --layer-ref style-analyst +ti fs mount-file-system --mount-path ./peek/v5 --driver fuse --layer-ref style-analyst --checkpoint-id v5 +``` + +The first mount writes to the active layer. The checkpoint mount is read-only. To continue from old history, fork a new writable child from that checkpoint instead of writing through the checkpoint mount: + +```bash +ti fs fork-layer --parent-layer-ref style-analyst --layer-name from-v5 --checkpoint-id v5 --actor-id hermes +ti fs mount-file-system --mount-path ./from-v5 --driver fuse --layer-ref from-v5 +``` + +Rejected leaf timelines can be abandoned, and a selected timeline can be committed to the base file system: + +```bash +ti fs delete-layer --layer-ref style-brief +ti fs delete-layer --layer-ref style-longform +ti fs commit-layer --layer-id from-v5 +``` + +## Product Semantics + +- A root layer overlays the live base file system under `base_root_path`. +- Fork pins the parent overlay at its current tip, or at one explicit parent checkpoint. It does not snapshot the live base file system. +- Parent overlay writes after the pin are invisible to the child. +- Changes committed by another layer to the live base can still become visible through base fallback. Layer fork is copy-on-write history, not a fully isolated database snapshot. +- A child writes only to its own top layer. It never mutates its pinned parent. +- A checkpoint mount is always read-only. A writable historical continuation requires `fork-layer --checkpoint-id`. +- `delete-layer` logically abandons a layer. It does not mean physical data erasure. +- Deleting a layer with live descendants fails unless `--cascade` is explicitly supplied. Cascade abandons descendants before the selected layer. +- `commit-layer` remains the only operation in this workflow that publishes the effective layer view into the live base file system. +- `rollback-layer` retains its existing Drive9 meaning. This spec does not invent rollback-to-checkpoint, merge, rebase, squash, or commit-into-parent behavior. + +## Command Surface + +### `ti fs fork-layer` + +```text +ti fs fork-layer + --parent-layer-ref + [--layer-id ] + [--layer-name ] + [--checkpoint-id ] + [--actor-id ] + [--dry-run] + [global options] +``` + +- `--parent-layer-ref` is required and accepts any layer reference supported by the Drive9 public CLI: layer ID, unique layer name, or supported tag reference. +- `--layer-id` optionally requests a stable child ID. When omitted, the service generates one. +- `--layer-name` optionally assigns a human-readable child name. +- `--checkpoint-id` pins the child to a checkpoint owned by the parent. When omitted, the child pins the serialized parent tip. +- `--actor-id` records the child owner or agent identity. +- The command maps to `ti-drive9 fs layer fork --json [--id ...] [--name ...] [--checkpoint ...] [--actor ...] `. Drive9 uses Go `flag.FlagSet`, so every companion flag must precede the positional parent reference. +- JSON output is the child layer object returned by Drive9, including ancestry fields. Text output renders the child layer ID, name, state, parent layer ID, origin sequence/checkpoint, depth, root layer ID, and base root path. +- The command requires `authz.FSFileWrite` and supports `--dry-run`. + +### `ti fs list-layer-chain` + +```text +ti fs list-layer-chain + --layer-ref + [global options] +``` + +- `--layer-ref` is required and accepts a Drive9 layer reference. +- The command maps to `ti-drive9 fs layer chain --json `. The JSON flag must precede the positional layer reference. +- JSON output is `{ "chain": [...] }`, ordered from root to the selected tip. +- Text output uses stable columns for layer ID, name, state, depth, parent layer ID, origin sequence, limit sequence, origin checkpoint ID, and base root path. +- The command requires `authz.FSFileRead`, is read-only, and rejects `--dry-run`. + +### `ti fs delete-layer` + +```text +ti fs delete-layer + --layer-ref + [--cascade] + [--dry-run] + [global options] +``` + +- `--layer-ref` is required and accepts a Drive9 layer reference. +- `--cascade` is false by default. There is no confirmation-name flag and no prompt. +- The command maps to `ti-drive9 fs layer delete [--cascade] `. The cascade flag must precede the positional layer reference and is added only when explicitly set. +- Because Drive9 returns only `ok`, `ti` returns its own structured result with `operation`, `layer_ref`, `status: "abandoned"`, and `cascade`. +- The command requires `authz.FSFileWrite` and supports `--dry-run`. +- A descendant conflict must remain a non-zero actionable error. `ti` must not retry with cascade automatically. + +### Layer-aware mount + +Extend `ti fs mount-file-system` and its `mount` alias: + +```text +ti fs mount-file-system + --mount-path + [--layer-ref ] + [--checkpoint-id ] + [existing mount options] +``` + +- `--layer-ref` maps to Drive9 `mount --layer`. +- `--checkpoint-id` maps to Drive9 `mount --checkpoint` and requires `--layer-ref`. +- Any layer-aware mount requires FUSE. An explicit `--driver webdav` fails locally. On a platform where `--driver auto` resolves to WebDAV, fail with an actionable message telling the user to install/enable FUSE and pass `--driver fuse`; do not silently switch drivers. +- A checkpoint mount is forced read-only. Supplying an explicit contradictory writable setting fails locally rather than relying on Drive9 to reinterpret it. +- The mount result adds `layer_ref`, `checkpoint_id`, and `read_only` when applicable. Mount locator state remains non-secret and records enough routing information for drain and unmount; unmount still selects the runtime only by `--mount-path`. +- Existing flat mounts are unchanged when both new flags are absent. + +## Reference And Identifier Rules + +Existing commands keep their current `--layer-id` flags in this spec to avoid an unrelated breaking change. New operations use `--layer-ref` only where the Drive9 contract intentionally resolves an ID, name, or tag reference. Help text must not call a reference an ID. + +`ti` passes references as opaque non-empty values after rejecting control characters and path separators that could alter companion argument or API-path interpretation. It does not resolve names or tags locally, cache an ID mapping, or infer a current layer. + +Checkpoint IDs are explicit opaque identifiers. There is no checkpoint-list command in the current Drive9 public CLI, so examples use stable caller-assigned checkpoint IDs. + +## Output Models + +Extend the local Drive9-compatible layer DTOs with fields already returned by the public companion: + +- `parent_layer_id` +- `origin_seq` +- `origin_checkpoint_id` +- `root_layer_id` +- `depth` +- `origin` + +Add: + +- `ForkLayerOptions` and the existing `LayerResult` child response; +- `ListLayerChainOptions`, `LayerChainResult`, and `LayerChainFrame`; +- `DeleteLayerOptions` and `DeleteLayerResult`; +- layer/checkpoint/read-only fields on `MountResult`. + +All structured results support JSON/text rendering and JMESPath `--query` through the existing output pipeline. No service package prints directly to stdout. + +## Implementation Design + +### CLI wiring + +`internal/cli/commands.go` registers the three commands under `ti fs`, parses long flags, declares exactly one permission per command, and routes normal and dry-run execution through the shared command path. The command tree remains two levels. + +The same file extends mount option parsing and local validation. Unix-style aliases are not added for layer lifecycle commands. + +### Filesystem service + +`internal/fs/layer.go` owns option/result types and delegates normal execution to companion adapter methods: + +- `drive9ForkLayer` +- `drive9ListLayerChain` +- `drive9DeleteLayer` + +`internal/fs/drive9_companion.go` constructs arguments, invokes the isolated per-resource companion runtime, decodes JSON for fork/chain, and converts delete's `ok` into a structured `ti` result. It must not import or execute code under `ref/drive9`. + +Mount argument construction appends `--layer` and `--checkpoint` before remote/local positional arguments. The existing runner continues to sanitize inherited `DRIVE9_*` variables and supplies the selected FS token, endpoint, region, and isolated Drive9 home. + +### Resource and credential selection + +The new commands use the same explicit Filesystem selection and token precedence as other data-plane commands: + +1. explicit `--file-system-id` or `TI_FS_FILE_SYSTEM_ID`; +2. ID embedded in an explicit `--fs-token` or `TI_FS_TOKEN`; +3. otherwise fail before starting the companion. + +When an ID is selected without an explicit token, the corresponding profile-scoped local credential may supply the token. No default Filesystem is introduced. + +### Dry run + +`fork-layer` and `delete-layer` support dry run without invoking a companion mutation. Dry run validates profile/environment selection, Filesystem ID/token agreement, endpoint resolution, companion availability/capability, layer/checkpoint input, and permission declaration. Its request summary describes the equivalent public Drive9 operation without including FS tokens or local paths. + +Mount dry run includes the selected driver, layer ref, checkpoint ID, and effective read-only state. It never starts a mount process. + +## Companion And Backend Contract + +The implementation depends on public Drive9 behavior introduced by `mem9-ai/drive9` commit `99aceec47c949c1b6f74233109cbdf8e10fb9d56` or a later compatible release: + +- `drive9 fs layer fork` +- `drive9 fs layer chain` +- `drive9 fs layer delete` +- `drive9 mount --layer` +- `drive9 mount --checkpoint` + +Corresponding hosted endpoints are: + +- `POST /v1/layers/{parentRef}/fork` +- `GET /v1/layers/{layerRef}/chain` +- `DELETE /v1/layers/{layerRef}?cascade=true|false` + +The companion remains the production integration boundary. Endpoint paths are documented for contract verification and dry-run descriptions; `ti` must not add a native HTTP fallback. + +At startup, affected commands must detect an older companion and fail with an actionable incompatibility error rather than surfacing `unknown fs layer command`, silently changing behavior, or attempting internal endpoints. Capability detection can use shared companion metadata plus a command-surface probe and should be cached only for the current process. + +## Dependencies And Portability + +- No new Go module dependency is required. +- No cgo dependency is added. +- Fork, chain, and delete are available on every platform supported by the companion. +- Layer mounts require FUSE. Flat WebDAV mounting remains available but cannot expose a layer or checkpoint view. +- The release archive continues to contain `ti` and `ti-drive9`. `ref/drive9` remains excluded from build, tests, packaging, and runtime. + +## Tests + +### Unit tests + +Fake-companion tests must verify exact argument order and output decoding for: + +- tip fork with a generated child ID; +- checkpoint fork with explicit child ID/name/actor; +- root-to-tip chain JSON and text output; +- leaf delete; +- cascade delete; +- descendant conflict propagation; +- layer mount and checkpoint mount argument mapping; +- checkpoint requires layer; +- checkpoint forces read-only; +- WebDAV rejects layer/checkpoint locally; +- old companion capability failure; +- secrets never appear in results, errors, logs, or dry-run output. + +Black-box `make e2e` tests must cover help/required flags, JSON/text/query behavior, dry-run behavior, config-free `TI_FS_TOKEN` selection, profile-stored token selection, aliases for mount only, and unchanged flat mount behavior. + +### Live e2e + +`make live-e2e-fs` adds one real lifecycle using only resources created by or explicitly selected for that test: + +1. create a root layer and write seed content into its overlay; +2. checkpoint it with a stable unique ID; +3. fork two children from the same checkpoint; +4. verify both chains and pinned ancestry; +5. write different content through each child and verify isolation; +6. mount one child through FUSE and verify POSIX writes appear in that child; +7. mount the parent checkpoint read-only and verify writes fail; +8. delete one leaf and verify its state is abandoned; +9. verify deleting a parent with a live descendant fails, then validate explicit cascade on test-owned layers; +10. commit the selected child and verify content appears in the base file system; +11. clean up every test-owned mount and remaining layer without touching pre-existing layers. + +If FUSE is unavailable on the CI host, non-mount fork/chain/delete tests still run and the mount portion reports an explicit platform skip. A release is not accepted solely on fake-companion tests. + +## Release Gates + +This spec is complete only when all of the following are true: + +1. The official Drive9 release endpoint used by the ti installer publishes a companion at `99aceec4` or later with the required public commands. +2. Installer and updater tests verify the downloaded companion command surface, not only its checksum. +3. An authenticated hosted test passes fork, chain, delete, writable layer mount, and read-only checkpoint mount. +4. `make test`, `make e2e`, and `make live-e2e-fs` pass with the release companion. +5. README, AGENTS, and PingCAP command/example documentation are updated in the same product change. + +As of 2026-08-19, source implementation is available, but `https://drive9.ai/releases/drive9-darwin-arm64` reports commit `dac2d626` and does not yet expose `fork|chain|delete`. Coding can proceed against the confirmed public contract, but release completion remains blocked until the companion artifact is published and hosted behavior passes authenticated live verification. + +## Out Of Scope + +- Implementing or copying Drive9 layer semantics into `ti`. +- Native HTTP fallback for layer operations. +- Tenant-level Drive9 fork. +- Snapshotting the live base file system at fork time. +- Layer merge, rebase, squash, commit-into-parent, or rollback-to-checkpoint. +- A local current-layer context or default layer. +- Automatic cascade deletion. +- WebDAV layer mounts. diff --git a/docs/spec/0032-homebrew-and-scoop-distribution.md b/docs/spec/0033-homebrew-and-scoop-distribution.md similarity index 100% rename from docs/spec/0032-homebrew-and-scoop-distribution.md rename to docs/spec/0033-homebrew-and-scoop-distribution.md diff --git a/docs/spec/0033-serverless-function-deployment.md b/docs/spec/0034-serverless-function-deployment.md similarity index 100% rename from docs/spec/0033-serverless-function-deployment.md rename to docs/spec/0034-serverless-function-deployment.md diff --git a/docs/spec/done/0012-install-and-update-distribution.md b/docs/spec/done/0012-install-and-update-distribution.md index c1ea241..8a602c7 100644 --- a/docs/spec/done/0012-install-and-update-distribution.md +++ b/docs/spec/done/0012-install-and-update-distribution.md @@ -2,7 +2,7 @@ ## Goal -Make `tdc` installable and updatable through deterministic GitHub Releases artifacts. The MVP channel is GoReleaser plus GitHub Releases, with shell and PowerShell installers. Homebrew and Scoop are intentionally deferred to `docs/spec/0032-homebrew-and-scoop-distribution.md`. +Make `tdc` installable and updatable through deterministic GitHub Releases artifacts. The MVP channel is GoReleaser plus GitHub Releases, with shell and PowerShell installers. Homebrew and Scoop are intentionally deferred to `docs/spec/0033-homebrew-and-scoop-distribution.md`. ## User-facing Commands @@ -266,7 +266,7 @@ Installer scripts: - Silent auto-update. - Updating TiDB Cloud credentials or DB SQL credentials. - Config migrations that modify user config during update. -- Homebrew tap and Scoop bucket publishing. See `0032-homebrew-and-scoop-distribution.md`. +- Homebrew tap and Scoop bucket publishing. See `0033-homebrew-and-scoop-distribution.md`. - Linux apt/yum repositories. - Winget publishing. - Notarization or binary signing beyond SHA-256 checksums for MVP. diff --git a/ref/drive9 b/ref/drive9 index 645f5ce..99aceec 160000 --- a/ref/drive9 +++ b/ref/drive9 @@ -1 +1 @@ -Subproject commit 645f5ce8bb14bfcbae801eb90b641c890620db48 +Subproject commit 99aceec47c949c1b6f74233109cbdf8e10fb9d56 From 08356e566fc751eac46c6280e533d1cc721cc484 Mon Sep 17 00:00:00 2001 From: Cheese Date: Wed, 19 Aug 2026 16:33:05 +0800 Subject: [PATCH 3/3] feat(fs): add layer fork workflows --- AGENTS.md | 13 + Makefile | 2 +- README.md | 22 + .../0032-filesystem-layer-fork-workflows.md | 102 ++++- e2e/cli_test.go | 119 +++++ e2e/installer_test.go | 56 ++- e2e/live_layer_test.go | 428 ++++++++++++++++++ e2e/live_test.go | 46 +- e2e/testdata/fake-drive9.go | 36 ++ internal/api/fs/layer.go | 42 +- internal/cli/commands.go | 171 ++++++- internal/fs/drive9_companion.go | 154 ++++++- internal/fs/drive9_companion_test.go | 226 ++++++++- internal/fs/fscred/credential.go | 6 + internal/fs/fscred/credential_test.go | 19 + internal/fs/layer.go | 122 ++++- internal/fs/mount.go | 100 ++++ internal/fs/mountlocator/locator.go | 12 + internal/update/update.go | 70 +++ internal/update/update_test.go | 71 ++- scripts/install.ps1 | 24 + scripts/install.sh | 22 + 22 files changed, 1813 insertions(+), 50 deletions(-) rename docs/spec/{ => done}/0032-filesystem-layer-fork-workflows.md (61%) create mode 100644 e2e/live_layer_test.go diff --git a/AGENTS.md b/AGENTS.md index 93f4bd6..1a930a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,6 +77,8 @@ Implemented: `docs/spec/done/0018-fs-token-auth-and-config-free-access.md` - Server-backed Filesystem token lifecycle management from `docs/spec/done/0030-file-system-token-lifecycle-management.md` +- Filesystem layer fork, ancestry, deletion, and historical mount workflows + from `docs/spec/done/0032-filesystem-layer-fork-workflows.md` - install and update distribution from `docs/spec/done/0012-install-and-update-distribution.md` - English PingCAP Preview documentation from @@ -123,9 +125,12 @@ Implemented: - `ti fs find-files` - `ti fs create-layer` - `ti fs list-layers` +- `ti fs fork-layer` +- `ti fs list-layer-chain` - `ti fs describe-layer` - `ti fs diff-layer` - `ti fs create-layer-checkpoint` +- `ti fs delete-layer` - `ti fs rollback-layer` - `ti fs commit-layer` - `ti fs mount-file-system` @@ -584,10 +589,13 @@ Implemented command behavior: - `ti fs create-layer --layer-id layer-1 --base-root-path /workspace --layer-name task --durability-mode restore-safe --tag task=auth` - `ti fs list-layers` - `ti fs list-layers --output text` +- `ti fs fork-layer --parent-layer-ref layer-1 --layer-name child --checkpoint-id cp-1` +- `ti fs list-layer-chain --layer-ref child --output text` - `ti fs describe-layer --layer-id layer-1` - `ti fs diff-layer --layer-id layer-1` - `ti fs copy-file --from-local ./README.md --to-remote /workspace/layered.md --layer-id layer-1` - `ti fs create-layer-checkpoint --layer-id layer-1 --checkpoint-id cp-1 --label before-commit` +- `ti fs delete-layer --layer-ref child` - `ti fs rollback-layer --layer-id layer-1` - `ti fs commit-layer --layer-id layer-1` - `ti fs pack-file-system --local-root ~/.ti/local/fs/demo --remote-root /workspace --mount-profile portable` @@ -600,6 +608,8 @@ Implemented command behavior: - `ti fs mount-file-system --file-system-id --mount-path ./workspace --mount-profile portable --pack-path /` - `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --read-cache-size-mb 256 --read-cache-max-file-mb 16` - `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --cache-dir ~/.ti/cache/workspace --write-back-cache=false` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace --remote-path /workspace --driver fuse --layer-ref layer-1` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace-v1 --remote-path /workspace --driver fuse --layer-ref layer-1 --checkpoint-id cp-1` - `ti fs drain-file-system --mount-path ./workspace` - `ti fs drain-file-system --mount-path ./workspace --timeout 30s` - `ti fs unmount-file-system --mount-path ./workspace` @@ -671,9 +681,12 @@ Registered command surface: - `ti fs find-files` - `ti fs create-layer` - `ti fs list-layers` +- `ti fs fork-layer` +- `ti fs list-layer-chain` - `ti fs describe-layer` - `ti fs diff-layer` - `ti fs create-layer-checkpoint` +- `ti fs delete-layer` - `ti fs rollback-layer` - `ti fs commit-layer` - `ti fs pack-file-system` diff --git a/Makefile b/Makefile index b650c57..2d54554 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ live-e2e-db: build $(LIVE_E2E_RUN) -run '^TestLiveDB' live-e2e-fs: build - $(LIVE_E2E_RUN) -run '^TestLive(FSRemoteInventoryLifecycle|FSCommandSurface|FSFileSystemTokenLifecycle|FSConfigurationFreeAccess|FSDataPlaneLifecycle|FSMountRuntime|FSWebDAVMountRuntime)$$' + $(LIVE_E2E_RUN) -run '^TestLive(FSRemoteInventoryLifecycle|FSCommandSurface|FSFileSystemTokenLifecycle|FSConfigurationFreeAccess|FSDataPlaneLifecycle|FSLayerForkWorkflow|FSMountRuntime|FSWebDAVMountRuntime)$$' live-e2e-fs-git: build $(LIVE_E2E_RUN) -run '^TestLiveFSGit' diff --git a/README.md b/README.md index 1722d35..06f98da 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,25 @@ Automatic mounting uses FUSE on Linux and WebDAV on macOS and Windows. macOS use Mount commands start the companion runtime in the background, wait until the mount is ready, and then return a structured result. Use `ti fs unmount-file-system` or `ti fs-vault unmount-vault` to end a mount. The public CLI does not expose a foreground mount mode. +Filesystem layers can fork copy-on-write child timelines without copying a workspace. Layer and checkpoint mounts require FUSE; checkpoint mounts are always read-only. Drive9 does not support combining recursive copy with a layer, so seed a directory tree through a writable layer mount, drain it, and then create the checkpoint: + +```shell +export TI_FS_FILE_SYSTEM_ID="" +ti fs create-directory --path /research/q3-market +ti fs create-layer --base-root-path /research/q3-market --layer-name research-base +SEED_MOUNT="$(mktemp -d)" +ti fs mount-file-system --mount-path "$SEED_MOUNT" --remote-path /research/q3-market --driver fuse --layer-ref research-base +tar -C . -cf - . | tar -C "$SEED_MOUNT" -xf - +ti fs drain-file-system --mount-path "$SEED_MOUNT" +ti fs unmount-file-system --mount-path "$SEED_MOUNT" +rmdir "$SEED_MOUNT" +ti fs create-layer-checkpoint --layer-id research-base --checkpoint-id seed +ti fs fork-layer --parent-layer-ref research-base --layer-name experiment-a --checkpoint-id seed +ti fs list-layer-chain --layer-ref experiment-a --output text +``` + +Use `ti fs delete-layer --layer-ref experiment-a` to abandon a rejected leaf. Deleting a layer with live descendants fails unless `--cascade` is supplied explicitly. Layer names are not durable unique identifiers because abandoned layers remain visible; automation should capture returned layer IDs and use run-unique checkpoint IDs. + `ti fs list-file-systems` reads the region-scoped remote inventory through TiDB Cloud credentials. A profile can access multiple file systems, including resources created on another machine. Data-plane commands never infer a resource from the number of local credentials, so provide `--file-system-id` or set `TI_FS_FILE_SYSTEM_ID`: ```shell @@ -308,9 +327,12 @@ ti fs search-file-content ti fs find-files ti fs create-layer ti fs list-layers +ti fs fork-layer +ti fs list-layer-chain ti fs describe-layer ti fs diff-layer ti fs create-layer-checkpoint +ti fs delete-layer ti fs rollback-layer ti fs commit-layer ti fs pack-file-system diff --git a/docs/spec/0032-filesystem-layer-fork-workflows.md b/docs/spec/done/0032-filesystem-layer-fork-workflows.md similarity index 61% rename from docs/spec/0032-filesystem-layer-fork-workflows.md rename to docs/spec/done/0032-filesystem-layer-fork-workflows.md index f817b83..0864e7e 100644 --- a/docs/spec/0032-filesystem-layer-fork-workflows.md +++ b/docs/spec/done/0032-filesystem-layer-fork-workflows.md @@ -13,40 +13,81 @@ After this spec, a user can seed a workspace once and fork independent child lay ```bash ti fs create-directory --path /research/q3-market ti fs create-layer --base-root-path /research/q3-market --layer-name research-base --actor-id hermes --tag topic=q3-market -ti fs copy-file --from-local ./ --to-remote /research/q3-market/ --recursive --layer-id research-base -ti fs create-layer-checkpoint --layer-id research-base --checkpoint-id seed --label workspace-seed -ti fs fork-layer --parent-layer-ref research-base --layer-name style-brief --actor-id hermes-brief --checkpoint-id seed -ti fs fork-layer --parent-layer-ref research-base --layer-name style-longform --actor-id hermes-longform --checkpoint-id seed -ti fs fork-layer --parent-layer-ref research-base --layer-name style-analyst --actor-id hermes-analyst --checkpoint-id seed +SEED_MOUNT="$(mktemp -d)" +ti fs mount-file-system --mount-path "$SEED_MOUNT" --remote-path /research/q3-market --driver fuse --layer-ref research-base +tar -C . -cf - . | tar -C "$SEED_MOUNT" -xf - +ti fs drain-file-system --mount-path "$SEED_MOUNT" +ti fs unmount-file-system --mount-path "$SEED_MOUNT" +rmdir "$SEED_MOUNT" +SEED="$(ti fs create-layer-checkpoint --layer-id research-base --checkpoint-id seed --label workspace-seed --query checkpoint_id --output text)" + +ti fs fork-layer --parent-layer-ref research-base --layer-name style-brief --actor-id hermes-brief --checkpoint-id "$SEED" +ti fs fork-layer --parent-layer-ref research-base --layer-name style-longform --actor-id hermes-longform --checkpoint-id "$SEED" +ti fs fork-layer --parent-layer-ref research-base --layer-name style-analyst --actor-id hermes-analyst --checkpoint-id "$SEED" + +ti fs list-layers ti fs list-layer-chain --layer-ref style-analyst ``` +The seed tree is written through a writable FUSE layer mount and must not publish seed files to the live base file system. `ti` does not add support for combining `copy-file --recursive` with `--layer-id`; Drive9 rejects that combination. Users who need to seed a directory tree use the mounted POSIX view, then drain and unmount it before checkpointing. + Each child is a copy-on-write timeline pinned to the selected parent tip or checkpoint. Forking does not copy the file tree. Changes remain outside the base file system until `commit-layer` succeeds. -Users can mount one writable child and one historical checkpoint view: +Users can mount all three writable children over the same remote root and compare their independent results: + +```bash +mkdir -p ./runs/brief ./runs/longform ./runs/analyst +ti fs mount-file-system --mount-path ./runs/brief --remote-path /research/q3-market --driver fuse --layer-ref style-brief +ti fs mount-file-system --mount-path ./runs/longform --remote-path /research/q3-market --driver fuse --layer-ref style-longform +ti fs mount-file-system --mount-path ./runs/analyst --remote-path /research/q3-market --driver fuse --layer-ref style-analyst + +diff -u ./runs/brief/reports/report.md ./runs/analyst/reports/report.md +ti fs find-files --path /research/q3-market --layer-id style-brief +ti fs search-file-content --path /research/q3-market --pattern TAM --layer-id style-analyst +ti fs diff-layer --layer-id style-analyst +``` + +Before creating a checkpoint for writes made through a mount, drain that mount so all local dirty state has reached the remote layer. A checkpoint is a server-side boundary and cannot discover pending writes in a local mount process: ```bash -ti fs mount-file-system --mount-path ./runs/analyst --driver fuse --layer-ref style-analyst -ti fs mount-file-system --mount-path ./peek/v5 --driver fuse --layer-ref style-analyst --checkpoint-id v5 +ti fs drain-file-system --mount-path ./runs/analyst +ti fs create-layer-checkpoint --layer-id style-analyst --checkpoint-id v5 --label narrative-ok ``` -The first mount writes to the active layer. The checkpoint mount is read-only. To continue from old history, fork a new writable child from that checkpoint instead of writing through the checkpoint mount: +Users can then compare the already-mounted current tip with a historical checkpoint. Do not mount the same writable layer a second time; reuse `./runs/analyst` as the tip view. The checkpoint mount is read-only. To continue from old history, fork a new writable child from that checkpoint instead of writing through the checkpoint mount: ```bash +mkdir -p ./peek/v5 ./from-v5 +ti fs mount-file-system --mount-path ./peek/v5 --remote-path /research/q3-market --driver fuse --layer-ref style-analyst --checkpoint-id v5 + +diff -ru ./peek/v5 ./runs/analyst + ti fs fork-layer --parent-layer-ref style-analyst --layer-name from-v5 --checkpoint-id v5 --actor-id hermes -ti fs mount-file-system --mount-path ./from-v5 --driver fuse --layer-ref from-v5 +ti fs list-layer-chain --layer-ref from-v5 +ti fs mount-file-system --mount-path ./from-v5 --remote-path /research/q3-market --driver fuse --layer-ref from-v5 ``` Rejected leaf timelines can be abandoned, and a selected timeline can be committed to the base file system: ```bash +ti fs unmount-file-system --mount-path ./runs/brief +ti fs unmount-file-system --mount-path ./runs/longform ti fs delete-layer --layer-ref style-brief ti fs delete-layer --layer-ref style-longform + +ti fs drain-file-system --mount-path ./from-v5 +ti fs create-layer-checkpoint --layer-id from-v5 --checkpoint-id v5b1 --label rewrite-from-v5 +ti fs unmount-file-system --mount-path ./peek/v5 +ti fs unmount-file-system --mount-path ./runs/analyst +ti fs rollback-layer --layer-id style-analyst +ti fs unmount-file-system --mount-path ./from-v5 ti fs commit-layer --layer-id from-v5 ``` +Rolling back the abandoned parent does not invalidate the active child pin. The child remains readable and writable and can commit its effective chain view to the live base file system. + ## Product Semantics - A root layer overlays the live base file system under `base_root_path`. @@ -54,7 +95,10 @@ ti fs commit-layer --layer-id from-v5 - Parent overlay writes after the pin are invisible to the child. - Changes committed by another layer to the live base can still become visible through base fallback. Layer fork is copy-on-write history, not a fully isolated database snapshot. - A child writes only to its own top layer. It never mutates its pinned parent. +- Recursive `copy-file` and `--layer-id` remain mutually exclusive. Directory-tree seeding uses a writable FUSE layer mount; `ti` must not implement a separate recursive layer data plane or silently fall back to the live base file system. - A checkpoint mount is always read-only. A writable historical continuation requires `fork-layer --checkpoint-id`. +- A checkpoint includes only entries already accepted by the service. Call `drain-file-system` before checkpointing writes made through a live FUSE mount; checkpoint creation does not drain another local process. +- Do not mount one writable layer at multiple local paths concurrently. Reuse its existing mount or unmount it before mounting the same layer elsewhere. Historical checkpoint views are separate read-only mounts. - `delete-layer` logically abandons a layer. It does not mean physical data erasure. - Deleting a layer with live descendants fails unless `--cascade` is explicitly supplied. Cascade abandons descendants before the selected layer. - `commit-layer` remains the only operation in this workflow that publishes the effective layer view into the live base file system. @@ -140,7 +184,9 @@ Existing commands keep their current `--layer-id` flags in this spec to avoid an `ti` passes references as opaque non-empty values after rejecting control characters and path separators that could alter companion argument or API-path interpretation. It does not resolve names or tags locally, cache an ID mapping, or infer a current layer. -Checkpoint IDs are explicit opaque identifiers. There is no checkpoint-list command in the current Drive9 public CLI, so examples use stable caller-assigned checkpoint IDs. +Layer names are references, not unique durable identifiers. Logical deletion leaves an `abandoned` layer visible, and abandoned layers still participate in name resolution. The fixed names in examples are for readability; automation must capture and use returned layer IDs or generate run-unique names so a repeated workflow cannot become ambiguous. + +Checkpoint IDs are explicit opaque identifiers and are unique within one File System, not merely within one layer. There is no checkpoint-list command in the current Drive9 public CLI, so examples use readable caller-assigned IDs. Tests and repeatable automation must namespace those IDs per run and retain them with their owning layer ID. ## Output Models @@ -244,6 +290,7 @@ Fake-companion tests must verify exact argument order and output decoding for: - WebDAV rejects layer/checkpoint locally; - old companion capability failure; - secrets never appear in results, errors, logs, or dry-run output. +- `copy-file --recursive --layer-id` is rejected locally with an actionable message directing users to a writable FUSE layer mount. Black-box `make e2e` tests must cover help/required flags, JSON/text/query behavior, dry-run behavior, config-free `TI_FS_TOKEN` selection, profile-stored token selection, aliases for mount only, and unchanged flat mount behavior. @@ -251,17 +298,19 @@ Black-box `make e2e` tests must cover help/required flags, JSON/text/query behav `make live-e2e-fs` adds one real lifecycle using only resources created by or explicitly selected for that test: -1. create a root layer and write seed content into its overlay; -2. checkpoint it with a stable unique ID; -3. fork two children from the same checkpoint; -4. verify both chains and pinned ancestry; -5. write different content through each child and verify isolation; -6. mount one child through FUSE and verify POSIX writes appear in that child; -7. mount the parent checkpoint read-only and verify writes fail; -8. delete one leaf and verify its state is abandoned; -9. verify deleting a parent with a live descendant fails, then validate explicit cascade on test-owned layers; -10. commit the selected child and verify content appears in the base file system; -11. clean up every test-owned mount and remaining layer without touching pre-existing layers. +1. create an empty live base root and a root layer over it; +2. mount the root layer through FUSE, copy a local tree through the POSIX view, drain and unmount it, and verify the files remain absent from the live base; +3. checkpoint it with a stable unique ID and fork three children from that checkpoint; +4. verify all three chains have the same pinned ancestry and that later parent writes are invisible; +5. mount all three children through FUSE at the same explicit remote root, write different reports, drain the mounts, and verify isolation through POSIX reads, layer-aware find/grep, and layer diff; +6. delete two leaf children, verify each remains queryable with state `abandoned`, and verify layer listing exposes their state; +7. continue writing through the selected child, drain before each stable checkpoint, and create at least three checkpoints including `v5` and a later tip; +8. keep one writable mount for the selected child tip, mount `v5` read-only at the same remote root, verify the historical view differs and rejects writes, and never create a second writable mount for the same layer; +9. fork `from-v5`, verify its root-to-tip chain, mount it writable, write and drain new content, and checkpoint it; +10. unmount the old selected parent, roll it back while `from-v5` remains active, and verify the child can still read its pinned parent state and accept a new write; +11. verify deleting a parent with a live descendant fails, then validate explicit cascade separately on test-owned layers; +12. drain and unmount `from-v5`, commit it, and verify only the selected effective result appears in the live base file system; +13. clean up every test-owned mount and remaining layer without touching pre-existing layers. If FUSE is unavailable on the CI host, non-mount fork/chain/delete tests still run and the mount portion reports an explicit platform skip. A release is not accepted solely on fake-companion tests. @@ -269,17 +318,20 @@ If FUSE is unavailable on the CI host, non-mount fork/chain/delete tests still r This spec is complete only when all of the following are true: -1. The official Drive9 release endpoint used by the ti installer publishes a companion at `99aceec4` or later with the required public commands. +1. The official Drive9 release endpoint used by the ti installer publishes a compatible companion with fork, chain, delete, and layer/checkpoint mount support. 2. Installer and updater tests verify the downloaded companion command surface, not only its checksum. -3. An authenticated hosted test passes fork, chain, delete, writable layer mount, and read-only checkpoint mount. +3. An authenticated hosted test passes FUSE-mounted layer seed, fork, chain, delete, writable layer mount, read-only checkpoint mount, parent rollback with a live pinned child, and child commit. 4. `make test`, `make e2e`, and `make live-e2e-fs` pass with the release companion. 5. README, AGENTS, and PingCAP command/example documentation are updated in the same product change. -As of 2026-08-19, source implementation is available, but `https://drive9.ai/releases/drive9-darwin-arm64` reports commit `dac2d626` and does not yet expose `fork|chain|delete`. Coding can proceed against the confirmed public contract, but release completion remains blocked until the companion artifact is published and hosted behavior passes authenticated live verification. +As of 2026-08-19, the official artifact at `https://drive9.ai/releases/drive9-darwin-arm64` reports Drive9 commit `99aceec47c949c1b6f74233109cbdf8e10fb9d56` and exposes fork, chain, delete, and layer/checkpoint mount. It explicitly rejects recursive copy combined with `--layer`, which this spec intentionally leaves unsupported. The ti installer and updater execute the staged companion before replacing any installed binary and reject an artifact that does not expose the complete required surface. + +An authenticated `make live-e2e-fs` run against `aws-ap-southeast-1`, forced to use that downloaded official artifact, passed the complete layer workflow without skips or client-side fallback semantics. It created and FUSE-seeded a root layer, checkpointed and forked three isolated children, inspected pinned ancestry, abandoned rejected leaves, compared writable tip and read-only historical mounts, forked from an older checkpoint, validated descendant deletion safeguards, rolled back a parent with a live pinned child, and committed only the selected child to the base file system. The same run also passed remote inventory, command surface, token lifecycle, data-plane, ordinary FUSE mount, configuration-free access, and WebDAV coverage. `make test`, `make e2e`, `go vet ./...`, installer shell validation, and repository diff checks pass. ## Out Of Scope - Implementing or copying Drive9 layer semantics into `ti`. +- Supporting `copy-file --recursive` together with `--layer-id`; seed directory trees through a writable FUSE layer mount instead. - Native HTTP fallback for layer operations. - Tenant-level Drive9 fork. - Snapshotting the live base file system at fork time. diff --git a/e2e/cli_test.go b/e2e/cli_test.go index 9532023..c2d9ca1 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -794,6 +794,15 @@ func TestFSRemoteInventoryAndIDCredentialSelectionAcrossCommandFamilies(t *testi git.wantExitCode(0) mount := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "mount-file-system", "--file-system-id", "tenant-aws-us-west-2", "--mount-path", filepath.Join(home, "mount")) mount.wantExitCode(0) + profileFork := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "fork-layer", "--file-system-id", "tenant-aws-us-west-2", "--parent-layer-ref", "parent-id", "--layer-id", "profile-child") + profileFork.wantExitCode(0) + profileFork.wantStdoutContains(`"layer_id": "child-id"`) + profileChain := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "list-layer-chain", "--file-system-id", "tenant-aws-us-west-2", "--layer-ref", "profile-child") + profileChain.wantExitCode(0) + profileChain.wantStdoutContains(`"chain"`) + profileDeleteLayer := runTIWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "delete-layer", "--file-system-id", "tenant-aws-us-west-2", "--layer-ref", "profile-child") + profileDeleteLayer.wantExitCode(0) + profileDeleteLayer.wantStdoutContains(`"status": "abandoned"`) calls := readFakeDrive9Calls(t, recordPath) for _, call := range calls { @@ -803,6 +812,9 @@ func TestFSRemoteInventoryAndIDCredentialSelectionAcrossCommandFamilies(t *testi } assertFakeDrive9Call(t, calls, []string{"fs", "ls"}, drive9TestToken("tenant-aws-us-west-2"), home, "stage", "tenant-aws-us-west-2", westControl.URL(), "aws-us-west-2") assertFakeDrive9Call(t, calls, []string{"vault", "ls"}, drive9TestToken("tenant-aws-us-east-1"), home, "stage", "tenant-aws-us-east-1", eastControl.URL(), "aws-us-east-1") + assertFakeDrive9Call(t, calls, []string{"fs", "layer", "fork"}, drive9TestToken("tenant-aws-us-west-2"), home, "stage", "tenant-aws-us-west-2", westControl.URL(), "aws-us-west-2") + assertFakeDrive9Call(t, calls, []string{"fs", "layer", "chain"}, drive9TestToken("tenant-aws-us-west-2"), home, "stage", "tenant-aws-us-west-2", westControl.URL(), "aws-us-west-2") + assertFakeDrive9Call(t, calls, []string{"fs", "layer", "delete"}, drive9TestToken("tenant-aws-us-west-2"), home, "stage", "tenant-aws-us-west-2", westControl.URL(), "aws-us-west-2") if !eastControl.hasRequest(http.MethodPost, "/v1/admin/tenants") || !eastControl.hasRequest(http.MethodGet, "/v1/admin/tenants", "display_name=agent", "label=environment%3D%3Dproduction") || !westControl.hasRequest(http.MethodGet, "/v1/admin/tenants/tenant-aws-us-west-2") { @@ -955,6 +967,93 @@ func TestFSConfigurationFreeAccess(t *testing.T) { assertFakeDrive9Call(t, calls, []string{"umount"}, "", home, "default", "tenant-sandbox", "https://fs-east.test", "aws-us-east-1") } +func TestFSLayerForkWorkflowCommands(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fake companion build path is covered by unit tests on Windows") + } + bin := tiBinary(t) + home := t.TempDir() + companion := filepath.Join(t.TempDir(), "ti-drive9") + build := exec.Command("go", "build", "-o", companion, "./testdata/fake-drive9.go") + build.Dir = "." + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build fake Drive9 companion: %v\n%s", err, output) + } + recordPath := filepath.Join(t.TempDir(), "calls.jsonl") + manifestServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `{"service":"drive9","regions":[{"region_code":"aws-us-east-1","mode":"tidb_cloud_native","server_url":"https://fs-east.test","cloud_provider":"aws","tidb_region":"us-east-1"}]}`) + })) + defer manifestServer.Close() + env := []string{ + "HOME=" + home, + "TI_DRIVE9_BIN=" + companion, + "FAKE_DRIVE9_RECORD=" + recordPath, + "TI_ALLOW_TEST_ENDPOINTS=1", + "TI_TEST_FS_MANIFEST_URL=" + manifestServer.URL, + "TI_FS_TOKEN=" + drive9TestToken("tenant-layer"), + "TI_REGION_CODE=aws-us-east-1", + } + + fork := runTIWithInput(t, bin, "", env, "fs", "fork-layer", "--parent-layer-ref", "research-base", "--layer-id", "child-id", "--layer-name", "child", "--checkpoint-id", "seed", "--actor-id", "agent") + fork.wantExitCode(0) + fork.wantStdoutContains(`"parent_layer_id": "parent-id"`) + fork.wantStdoutNotContains("drive9_") + + chain := runTIWithInput(t, bin, "", env, "fs", "list-layer-chain", "--layer-ref", "child-id", "--output", "text") + chain.wantExitCode(0) + chain.wantStdoutContains("LAYER_ID") + chain.wantStdoutContains("root-id") + chain.wantStdoutContains("child-id") + + queried := runTIWithInput(t, bin, "", env, "fs", "list-layer-chain", "--layer-ref", "child-id", "--query", "chain[].layer_id") + queried.wantExitCode(0) + queried.wantStdoutContains("root-id") + + dryRunDelete := runTIWithInput(t, bin, "", env, "fs", "delete-layer", "--layer-ref", "child-id", "--cascade", "--dry-run") + dryRunDelete.wantExitCode(0) + dryRunDelete.wantStdoutContains(`"operation": "delete_layer"`) + dryRunDelete.wantStdoutContains(`"companion_capability"`) + dryRunDelete.wantStdoutNotContains(drive9TestToken("tenant-layer")) + + deleted := runTIWithInput(t, bin, "", env, "fs", "delete-layer", "--layer-ref", "child-id", "--cascade", "--output", "text") + deleted.wantExitCode(0) + deleted.wantStdoutContains("status=abandoned") + + recursive := runTIWithInput(t, bin, "", env, "fs", "copy-file", "--from-local", ".", "--to-remote", "/workspace", "--recursive", "--layer-id", "child-id") + recursive.wantExitCode(2) + recursive.wantStderrContains("--recursive cannot be combined with --layer-id") + + checkpointWithoutLayer := runTIWithInput(t, bin, "", env, "fs", "mount-file-system", "--mount-path", filepath.Join(home, "missing-layer"), "--driver", "fuse", "--checkpoint-id", "seed") + checkpointWithoutLayer.wantExitCode(2) + checkpointWithoutLayer.wantStderrContains("--checkpoint-id requires --layer-ref") + + webdavLayer := runTIWithInput(t, bin, "", env, "fs", "mount-file-system", "--mount-path", filepath.Join(home, "webdav"), "--driver", "webdav", "--layer-ref", "child-id") + webdavLayer.wantExitCode(2) + webdavLayer.wantStderrContains("layer and checkpoint mounts require FUSE") + + mountPath := filepath.Join(home, "checkpoint") + mounted := runTIWithInput(t, bin, "", env, "fs", "mount-file-system", "--mount-path", mountPath, "--remote-path", "/workspace", "--driver", "fuse", "--layer-ref", "child-id", "--checkpoint-id", "seed") + mounted.wantExitCode(0) + mounted.wantStdoutContains(`"read_only": true`) + mounted.wantStdoutContains(`"write_back_cache": false`) + mounted.wantStdoutContains(`"checkpoint_id": "seed"`) + + explicitWritable := runTIWithInput(t, bin, "", env, "fs", "mount-file-system", "--mount-path", filepath.Join(home, "writable-checkpoint"), "--driver", "fuse", "--layer-ref", "child-id", "--checkpoint-id", "seed", "--read-only=false") + explicitWritable.wantExitCode(2) + explicitWritable.wantStderrContains("checkpoint mounts are read-only") + explicitWriteBack := runTIWithInput(t, bin, "", env, "fs", "mount-file-system", "--mount-path", filepath.Join(home, "write-back-checkpoint"), "--driver", "fuse", "--layer-ref", "child-id", "--checkpoint-id", "seed", "--write-back-cache=true") + explicitWriteBack.wantExitCode(2) + explicitWriteBack.wantStderrContains("checkpoint mounts are read-only") + unmounted := runTIWithInput(t, bin, "", env, "fs", "unmount-file-system", "--mount-path", mountPath) + unmounted.wantExitCode(0) + + calls := readFakeDrive9Calls(t, recordPath) + assertFakeDrive9Args(t, calls, []string{"fs", "layer", "fork", "--json", "--id", "child-id", "--name", "child", "--checkpoint", "seed", "--actor", "agent", "research-base"}) + assertFakeDrive9Args(t, calls, []string{"fs", "layer", "chain", "--json", "child-id"}) + assertFakeDrive9Args(t, calls, []string{"fs", "layer", "delete", "--cascade", "child-id"}) + assertFakeDrive9Args(t, calls, []string{"mount", "--mode", "fuse", "--read-only", "--layer", "child-id", "--checkpoint", "seed", "--cache-size", "128", "--read-cache-max-file-mb", "4", "--read-cache-ttl", "30s", ":/workspace", mountPath}) +} + func TestFSImportFileSystemToken(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("POSIX token-file permission checks are not available on Windows") @@ -1413,6 +1512,26 @@ func assertFakeDrive9Call(t *testing.T, calls []fakeDrive9Call, prefix []string, t.Fatalf("missing fake Drive9 call with prefix %v in %#v", prefix, calls) } +func assertFakeDrive9Args(t *testing.T, calls []fakeDrive9Call, want []string) { + t.Helper() + for _, call := range calls { + if len(call.Args) != len(want) { + continue + } + match := true + for i := range want { + if call.Args[i] != want[i] { + match = false + break + } + } + if match { + return + } + } + t.Fatalf("missing exact fake Drive9 args %v in %#v", want, calls) +} + func assertFakeDrive9TransientCall(t *testing.T, calls []fakeDrive9Call, prefix []string, apiKey, persistentHome, server, regionCode string) { t.Helper() for _, call := range calls { diff --git a/e2e/installer_test.go b/e2e/installer_test.go index ad6663c..c34e605 100644 --- a/e2e/installer_test.go +++ b/e2e/installer_test.go @@ -146,7 +146,8 @@ func TestUnixInstallerMigratesLegacyStateAndPreservesPathShadow(t *testing.T) { companionArtifact := fmt.Sprintf("drive9-%s-%s", runtime.GOOS, runtime.GOARCH) companion := filepath.Join(assetDir, companionArtifact) - writeE2EFile(t, companion, "#!/bin/sh\nexit 0\n", 0o755) + companionCalls := filepath.Join(root, "companion-calls") + writeE2EFile(t, companion, compatibleDrive9ShellFixture(companionCalls), 0o755) companionData, err := os.ReadFile(companion) if err != nil { t.Fatal(err) @@ -198,6 +199,15 @@ fi if strings.Contains(string(output), "ti fs companion installed to") { t.Fatalf("installer exposed the companion installation path:\n%s", output) } + companionCallData, err := os.ReadFile(companionCalls) + if err != nil { + t.Fatalf("read companion validation calls: %v", err) + } + for _, want := range []string{"fs layer help", "mount --help"} { + if !strings.Contains(string(companionCallData), want) { + t.Fatalf("installer did not validate companion command surface %q; calls:\n%s", want, companionCallData) + } + } for _, regionCode := range []string{"aws-us-east-1", "aws-ap-southeast-1", "aws-us-west-2", "alicloud-ap-southeast-1"} { if !strings.Contains(string(output), regionCode) { t.Fatalf("installer did not list ti fs region %q:\n%s", regionCode, output) @@ -224,6 +234,21 @@ fi } } +func compatibleDrive9ShellFixture(callLog string) string { + return fmt.Sprintf(`#!/bin/sh +printf '%%s\n' "$*" >> %q +if [ "$1 $2 $3" = "fs layer help" ]; then + printf 'usage: drive9 fs layer \n' + exit 0 +fi +if [ "$1 $2" = "mount --help" ]; then + printf ' -layer string\n -checkpoint string\n' + exit 0 +fi +printf 'compatible companion\n' +`, callLog) +} + func copyTestFile(t *testing.T, source, destination string, mode os.FileMode) { t.Helper() data, err := os.ReadFile(source) @@ -277,6 +302,35 @@ func TestInstallersUseProductOwnedFSRegionsAndHideCompanionPath(t *testing.T) { } } +func TestInstallersValidateCompanionCommandSurfaceBeforeInstallation(t *testing.T) { + shellBytes, err := os.ReadFile(filepath.Join("..", "scripts", "install.sh")) + if err != nil { + t.Fatal(err) + } + shell := string(shellBytes) + shellValidation := strings.LastIndex(shell, `validate_companion "${TMP_DIR}/${COMPANION_ARTIFACT}"`) + shellInstall := strings.LastIndex(shell, `install_file "${TMP_DIR}/${COMPANION_ARTIFACT}" "$COMPANION_TARGET"`) + if shellValidation < 0 || shellInstall < 0 || shellValidation >= shellInstall { + t.Fatal("install.sh must validate the staged companion before installing it") + } + + powerShellBytes, err := os.ReadFile(filepath.Join("..", "scripts", "install.ps1")) + if err != nil { + t.Fatal(err) + } + powerShell := string(powerShellBytes) + powerShellValidation := strings.LastIndex(powerShell, "Assert-CompanionCommandSurface $CompanionPath") + powerShellInstall := strings.LastIndex(powerShell, "Move-Item -Force -Path $CompanionPath -Destination $CompanionTarget") + if powerShellValidation < 0 || powerShellInstall < 0 || powerShellValidation >= powerShellInstall { + t.Fatal("install.ps1 must validate the staged companion before installing it") + } + for _, required := range []string{"fork", "chain", "delete", "layer", "checkpoint"} { + if !strings.Contains(shell, required) || !strings.Contains(powerShell, required) { + t.Fatalf("installers do not validate required companion surface %q", required) + } + } +} + func TestInstallersExplainTelemetryControls(t *testing.T) { for _, path := range []string{ filepath.Join("..", "scripts", "install.sh"), diff --git a/e2e/live_layer_test.go b/e2e/live_layer_test.go new file mode 100644 index 0000000..59c7328 --- /dev/null +++ b/e2e/live_layer_test.go @@ -0,0 +1,428 @@ +package e2e + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + apifs "github.com/tidbcloud/ti-cli/internal/api/fs" + "github.com/tidbcloud/ti-cli/internal/fs/mountdriver" +) + +func TestLiveFSLayerForkWorkflow(t *testing.T) { + requireLive(t) + if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + t.Skip("filesystem layer mount live e2e requires macOS or Linux FUSE") + } + driver, err := mountdriver.Resolve("fuse") + if err != nil { + t.Fatalf("resolve FUSE driver: %v", err) + } + if err := driver.CheckPrerequisites(); err != nil { + t.Skipf("filesystem layer non-mount operations are covered by TestLiveFSDataPlaneLifecycle; full layer workflow requires FUSE: %v", err) + } + + bin := tiBinary(t) + profileName := liveProfileName(t) + selected := ensureLiveFSResource(t, bin, profileName) + fileSystemID := selected.FSTenantID + if fileSystemID == "" { + t.Fatal("selected live filesystem has no file system ID") + } + suffix := fmt.Sprintf("%d-%d", os.Getpid(), time.Now().UnixNano()) + remoteRoot := "/ti-e2e-layer-fork-" + suffix + rootLayerID := "ti-e2e-root-" + suffix + rootCheckpointID := "ti-e2e-seed-" + suffix + + runFS := func(command string, args ...string) commandResult { + cliArgs := []string{"--profile", profileName, "fs", command, "--file-system-id", fileSystemID} + cliArgs = append(cliArgs, args...) + return runTI(t, bin, cliArgs...) + } + runMountControl := func(command string, args ...string) commandResult { + cliArgs := []string{"--profile", profileName, "fs", command} + cliArgs = append(cliArgs, args...) + return runTI(t, bin, cliArgs...) + } + + mounted := map[string]bool{} + rootCreated := false + defer func() { + for mountPath, active := range mounted { + if !active { + continue + } + cleanup := runMountControl("unmount-file-system", "--mount-path", mountPath, "--ignore-absent", "--force") + if cleanup.exitCode != 0 { + t.Logf("cleanup layer mount failed for %s: exit=%d stderr=%s", mountPath, cleanup.exitCode, strings.TrimSpace(cleanup.stderr)) + } + } + if rootCreated { + cleanupLayer := runFS("delete-layer", "--layer-ref", rootLayerID, "--cascade") + if cleanupLayer.exitCode != 0 && cleanupLayer.exitCode != 5 { + t.Logf("cleanup root layer failed for %s: exit=%d stderr=%s", rootLayerID, cleanupLayer.exitCode, strings.TrimSpace(cleanupLayer.stderr)) + rollbackLayer := runFS("rollback-layer", "--layer-id", rootLayerID) + if rollbackLayer.exitCode != 0 && rollbackLayer.exitCode != 5 { + t.Logf("cleanup root layer rollback failed for %s: exit=%d stderr=%s", rootLayerID, rollbackLayer.exitCode, strings.TrimSpace(rollbackLayer.stderr)) + } + } + } + cleanupRoot := runFS("delete-file", "--path", remoteRoot, "--recursive") + if cleanupRoot.exitCode != 0 && cleanupRoot.exitCode != 5 && !isLiveFSNotFound(cleanupRoot.stderr) { + t.Logf("cleanup layer workflow root failed for %s: exit=%d stderr=%s", remoteRoot, cleanupRoot.exitCode, strings.TrimSpace(cleanupRoot.stderr)) + } + }() + + unmount := func(mountPath string) { + t.Helper() + result := runMountControl("unmount-file-system", "--mount-path", mountPath) + result.wantExitCode(0) + result.wantStdoutContains(`"status": "unmounted"`) + mounted[mountPath] = false + } + drain := func(mountPath string) { + t.Helper() + result := runMountControl("drain-file-system", "--mount-path", mountPath, "--timeout", "60s") + result.wantExitCode(0) + result.wantStdoutContains(`"status": "drained"`) + } + mountLayer := func(layerRef, checkpointID, mountPath string) { + t.Helper() + args := []string{ + "--mount-path", mountPath, + "--remote-path", remoteRoot, + "--driver", "fuse", + "--layer-ref", layerRef, + "--ready-timeout", "60s", + } + if checkpointID != "" { + args = append(args, "--checkpoint-id", checkpointID) + } + result := runFS("mount-file-system", args...) + result.wantExitCode(0) + result.wantStdoutContains(`"status": "mounted"`) + result.wantStdoutContains(`"layer_ref": "` + layerRef + `"`) + if checkpointID != "" { + result.wantStdoutContains(`"checkpoint_id": "` + checkpointID + `"`) + result.wantStdoutContains(`"read_only": true`) + } + mounted[mountPath] = true + } + checkpoint := func(layerID, checkpointID, label string) { + t.Helper() + result := runFS( + "create-layer-checkpoint", + "--layer-id", layerID, + "--checkpoint-id", checkpointID, + "--label", label, + ) + result.wantExitCode(0) + result.wantStdoutContains(checkpointID) + } + fork := func(parentID, childID, checkpointID, name string) apifs.FSLayer { + t.Helper() + args := []string{ + "--parent-layer-ref", parentID, + "--layer-id", childID, + "--layer-name", name, + "--actor-id", "ti-live-e2e", + } + if checkpointID != "" { + args = append(args, "--checkpoint-id", checkpointID) + } + result := runFS("fork-layer", args...) + result.wantExitCode(0) + var layer apifs.FSLayer + if err := json.Unmarshal([]byte(result.stdout), &layer); err != nil { + t.Fatalf("decode forked layer %s: %v\n%s", childID, err, result.stdout) + } + if layer.LayerID != childID || layer.ParentLayerID != parentID { + t.Fatalf("forked layer identity mismatch: %+v", layer) + } + if checkpointID != "" && layer.OriginCheckpointID != checkpointID { + t.Fatalf("forked layer checkpoint = %q, want %q: %+v", layer.OriginCheckpointID, checkpointID, layer) + } + return layer + } + assertBaseAbsent := func(remotePath string) { + t.Helper() + result := runFS("read-file", "--path", remotePath) + if result.exitCode == 0 { + result.fail("base filesystem unexpectedly contains %s", remotePath) + } + if !isLiveFSNotFound(result.stderr) { + result.fail("base filesystem absence check for %s failed for an unexpected reason", remotePath) + } + } + waitBaseRead := func(remotePath, want string) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + var last commandResult + for { + last = runFS("read-file", "--path", remotePath) + if last.exitCode == 0 && last.stdout == want { + return + } + if time.Now().After(deadline) { + last.fail("timed out waiting for committed base file %s", remotePath) + } + time.Sleep(time.Second) + } + } + + createRoot := runFS("create-directory", "--path", remoteRoot, "--mode", "0755") + createRoot.wantExitCode(0) + createLayer := runFS( + "create-layer", + "--layer-id", rootLayerID, + "--base-root-path", remoteRoot, + "--layer-name", "research-base-"+suffix, + "--actor-id", "ti-live-e2e-seed", + "--tag", "test=ti-e2e", + "--tag", "run="+suffix, + ) + createLayer.wantExitCode(0) + rootCreated = true + + seedSource := filepath.Join(t.TempDir(), "seed") + if err := os.MkdirAll(filepath.Join(seedSource, "reports"), 0o755); err != nil { + t.Fatalf("create local seed reports: %v", err) + } + if err := os.MkdirAll(filepath.Join(seedSource, "data"), 0o755); err != nil { + t.Fatalf("create local seed data: %v", err) + } + seedReport := "seed report " + suffix + "\n" + seedData := "seed data " + suffix + "\n" + if err := os.WriteFile(filepath.Join(seedSource, "reports", "report.md"), []byte(seedReport), 0o644); err != nil { + t.Fatalf("write local seed report: %v", err) + } + if err := os.WriteFile(filepath.Join(seedSource, "data", "input.txt"), []byte(seedData), 0o644); err != nil { + t.Fatalf("write local seed data: %v", err) + } + seedMount := filepath.Join(t.TempDir(), "seed-mount") + if err := os.MkdirAll(seedMount, 0o755); err != nil { + t.Fatalf("create seed mount path: %v", err) + } + mountLayer(rootLayerID, "", seedMount) + copyLiveDirectoryTree(t, seedSource, seedMount) + drain(seedMount) + unmount(seedMount) + assertBaseAbsent(remoteRoot + "/reports/report.md") + assertBaseAbsent(remoteRoot + "/data/input.txt") + seedFind := runFS("find-files", "--path", remoteRoot, "--file-name-pattern", "report.md", "--layer-id", rootLayerID) + seedFind.wantExitCode(0) + seedFind.wantStdoutContains(remoteRoot + "/reports/report.md") + checkpoint(rootLayerID, rootCheckpointID, "workspace-seed") + + briefID := "ti-e2e-brief-" + suffix + longformID := "ti-e2e-longform-" + suffix + analystID := "ti-e2e-analyst-" + suffix + fork(rootLayerID, briefID, rootCheckpointID, "style-brief-"+suffix) + fork(rootLayerID, longformID, rootCheckpointID, "style-longform-"+suffix) + fork(rootLayerID, analystID, rootCheckpointID, "style-analyst-"+suffix) + + for _, childID := range []string{briefID, longformID, analystID} { + chainResult := runFS("list-layer-chain", "--layer-ref", childID) + chainResult.wantExitCode(0) + var chain struct { + Chain []apifs.FSLayerChainFrame `json:"chain"` + } + if err := json.Unmarshal([]byte(chainResult.stdout), &chain); err != nil { + t.Fatalf("decode layer chain for %s: %v\n%s", childID, err, chainResult.stdout) + } + if len(chain.Chain) != 2 || chain.Chain[0].LayerID != rootLayerID || chain.Chain[1].LayerID != childID || chain.Chain[1].OriginCheckpointID != rootCheckpointID { + t.Fatalf("unexpected pinned chain for %s: %+v", childID, chain.Chain) + } + } + + parentLatePath := remoteRoot + "/parent-late.txt" + parentLateFile := filepath.Join(t.TempDir(), "parent-late.txt") + if err := os.WriteFile(parentLateFile, []byte("late parent write\n"), 0o644); err != nil { + t.Fatalf("write parent-late source: %v", err) + } + parentLate := runFS("copy-file", "--from-local", parentLateFile, "--to-remote", parentLatePath, "--layer-id", rootLayerID) + parentLate.wantExitCode(0) + for _, childID := range []string{briefID, longformID, analystID} { + findLate := runFS("find-files", "--path", remoteRoot, "--file-name-pattern", "parent-late.txt", "--layer-id", childID) + findLate.wantExitCode(0) + findLate.wantStdoutNotContains(parentLatePath) + } + + briefMount := filepath.Join(t.TempDir(), "brief") + longformMount := filepath.Join(t.TempDir(), "longform") + analystMount := filepath.Join(t.TempDir(), "analyst") + for _, mountPath := range []string{briefMount, longformMount, analystMount} { + if err := os.MkdirAll(mountPath, 0o755); err != nil { + t.Fatalf("create child mount path: %v", err) + } + } + mountLayer(briefID, "", briefMount) + mountLayer(longformID, "", longformMount) + mountLayer(analystID, "", analystMount) + for _, mountPath := range []string{briefMount, longformMount, analystMount} { + waitLiveLocalFile(t, filepath.Join(mountPath, "reports", "report.md"), seedReport, 30*time.Second) + } + briefContent := "brief report " + suffix + "\n" + longformContent := "longform report " + suffix + "\n" + analystV1 := "analyst v1 TAM " + suffix + "\n" + if err := os.WriteFile(filepath.Join(briefMount, "reports", "brief.md"), []byte(briefContent), 0o644); err != nil { + t.Fatalf("write brief report: %v", err) + } + if err := os.WriteFile(filepath.Join(longformMount, "reports", "longform.md"), []byte(longformContent), 0o644); err != nil { + t.Fatalf("write longform report: %v", err) + } + if err := os.WriteFile(filepath.Join(analystMount, "reports", "report.md"), []byte(analystV1), 0o644); err != nil { + t.Fatalf("write analyst report: %v", err) + } + for _, mountPath := range []string{briefMount, longformMount, analystMount} { + drain(mountPath) + } + if data, err := os.ReadFile(filepath.Join(analystMount, "reports", "report.md")); err != nil || string(data) != analystV1 { + t.Fatalf("analyst mount report mismatch: %q, %v", data, err) + } + if _, err := os.Stat(filepath.Join(analystMount, "reports", "brief.md")); !os.IsNotExist(err) { + t.Fatalf("brief child write leaked into analyst child: %v", err) + } + waitLiveFSResult(t, bin, []string{"--profile", profileName, "fs", "search-file-content", "--file-system-id", fileSystemID, "--path", remoteRoot, "--pattern", "TAM", "--layer-id", analystID}, "report.md", 2*time.Minute, "search analyst layer") + analystDiff := runFS("diff-layer", "--layer-id", analystID) + analystDiff.wantExitCode(0) + analystDiff.wantStdoutContains(remoteRoot + "/reports/report.md") + + unmount(briefMount) + unmount(longformMount) + for _, childID := range []string{briefID, longformID} { + deleted := runFS("delete-layer", "--layer-ref", childID) + deleted.wantExitCode(0) + described := runFS("describe-layer", "--layer-id", childID) + described.wantExitCode(0) + described.wantStdoutContains(`"state": "abandoned"`) + } + listAbandoned := runFS("list-layers") + listAbandoned.wantExitCode(0) + listAbandoned.wantStdoutContains(briefID) + listAbandoned.wantStdoutContains(longformID) + + cpV1 := "ti-e2e-v1-" + suffix + cpV5 := "ti-e2e-v5-" + suffix + cpV7 := "ti-e2e-v7-" + suffix + drain(analystMount) + checkpoint(analystID, cpV1, "first-draft") + analystV5 := "analyst v5 narrative " + suffix + "\n" + if err := os.WriteFile(filepath.Join(analystMount, "reports", "report.md"), []byte(analystV5), 0o644); err != nil { + t.Fatalf("write analyst v5: %v", err) + } + drain(analystMount) + checkpoint(analystID, cpV5, "narrative-ok") + analystV7 := "analyst v7 latest " + suffix + "\n" + if err := os.WriteFile(filepath.Join(analystMount, "reports", "report.md"), []byte(analystV7), 0o644); err != nil { + t.Fatalf("write analyst v7: %v", err) + } + drain(analystMount) + checkpoint(analystID, cpV7, "latest-tip") + + v5Mount := filepath.Join(t.TempDir(), "v5") + if err := os.MkdirAll(v5Mount, 0o755); err != nil { + t.Fatalf("create v5 mount path: %v", err) + } + mountLayer(analystID, cpV5, v5Mount) + waitLiveLocalFile(t, filepath.Join(v5Mount, "reports", "report.md"), analystV5, 30*time.Second) + waitLiveLocalFile(t, filepath.Join(analystMount, "reports", "report.md"), analystV7, 30*time.Second) + if err := os.WriteFile(filepath.Join(v5Mount, "reports", "report.md"), []byte("must fail\n"), 0o644); err == nil { + t.Fatal("historical checkpoint mount accepted a write") + } + + fromV5ID := "ti-e2e-from-v5-" + suffix + fork(analystID, fromV5ID, cpV5, "from-v5-"+suffix) + fromV5Chain := runFS("list-layer-chain", "--layer-ref", fromV5ID) + fromV5Chain.wantExitCode(0) + fromV5Chain.wantStdoutContains(`"origin_checkpoint_id": "` + cpV5 + `"`) + fromV5Mount := filepath.Join(t.TempDir(), "from-v5") + if err := os.MkdirAll(fromV5Mount, 0o755); err != nil { + t.Fatalf("create from-v5 mount path: %v", err) + } + mountLayer(fromV5ID, "", fromV5Mount) + waitLiveLocalFile(t, filepath.Join(fromV5Mount, "reports", "report.md"), analystV5, 30*time.Second) + selectedContent := "selected rewrite from v5 " + suffix + "\n" + if err := os.WriteFile(filepath.Join(fromV5Mount, "reports", "report.md"), []byte(selectedContent), 0o644); err != nil { + t.Fatalf("write selected from-v5 report: %v", err) + } + drain(fromV5Mount) + checkpoint(fromV5ID, "ti-e2e-v5b1-"+suffix, "rewrite-from-v5") + + deleteParent := runFS("delete-layer", "--layer-ref", analystID) + if deleteParent.exitCode == 0 { + deleteParent.fail("deleting a parent with live descendant %s unexpectedly succeeded", fromV5ID) + } + if !strings.Contains(strings.ToLower(deleteParent.stderr), "descendant") { + deleteParent.fail("deleting a parent with live descendant %s failed for an unrelated reason", fromV5ID) + } + + cascadeParentID := "ti-e2e-cascade-parent-" + suffix + cascadeChildID := "ti-e2e-cascade-child-" + suffix + fork(rootLayerID, cascadeParentID, rootCheckpointID, "cascade-parent-"+suffix) + fork(cascadeParentID, cascadeChildID, "", "cascade-child-"+suffix) + cascadeDelete := runFS("delete-layer", "--layer-ref", cascadeParentID, "--cascade") + cascadeDelete.wantExitCode(0) + for _, layerID := range []string{cascadeParentID, cascadeChildID} { + described := runFS("describe-layer", "--layer-id", layerID) + described.wantExitCode(0) + described.wantStdoutContains(`"state": "abandoned"`) + } + + unmount(v5Mount) + unmount(analystMount) + rollbackParent := runFS("rollback-layer", "--layer-id", analystID) + rollbackParent.wantExitCode(0) + waitLiveLocalFile(t, filepath.Join(fromV5Mount, "reports", "report.md"), selectedContent, 30*time.Second) + continuedContent := "continued after parent rollback " + suffix + "\n" + if err := os.WriteFile(filepath.Join(fromV5Mount, "reports", "continued.md"), []byte(continuedContent), 0o644); err != nil { + t.Fatalf("write child after parent rollback: %v", err) + } + drain(fromV5Mount) + waitLiveLocalFile(t, filepath.Join(fromV5Mount, "reports", "continued.md"), continuedContent, 30*time.Second) + unmount(fromV5Mount) + + commitSelected := runFS("commit-layer", "--layer-id", fromV5ID) + commitSelected.wantExitCode(0) + waitBaseRead(remoteRoot+"/reports/report.md", selectedContent) + waitBaseRead(remoteRoot+"/reports/continued.md", continuedContent) + waitBaseRead(remoteRoot+"/data/input.txt", seedData) + assertBaseAbsent(remoteRoot + "/reports/brief.md") + assertBaseAbsent(remoteRoot + "/reports/longform.md") + assertBaseAbsent(parentLatePath) +} + +func copyLiveDirectoryTree(t *testing.T, sourceRoot, destinationRoot string) { + t.Helper() + if err := filepath.Walk(sourceRoot, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(sourceRoot, path) + if err != nil { + return err + } + if relative == "." { + return nil + } + destination := filepath.Join(destinationRoot, relative) + if info.IsDir() { + return os.MkdirAll(destination, info.Mode().Perm()) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("unsupported seed entry %s (%s)", path, info.Mode()) + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + return os.WriteFile(destination, data, info.Mode().Perm()) + }); err != nil { + t.Fatalf("copy seed tree through layer mount: %v", err) + } +} diff --git a/e2e/live_test.go b/e2e/live_test.go index fe4cc1e..371b524 100644 --- a/e2e/live_test.go +++ b/e2e/live_test.go @@ -84,6 +84,7 @@ func TestLiveFSRemoteInventoryLifecycle(t *testing.T) { bin := tiBinary(t) profileName := liveProfileName(t) + releaseAutoCreatedLiveFSResource(t, bin, profileName) displayName := fmt.Sprintf("ti-e2e-fs-%d-%d", os.Getpid(), time.Now().UnixNano()) labelKey := "ti-e2e-run" labelValue := fmt.Sprintf("run-%d-%d", os.Getpid(), time.Now().UnixNano()) @@ -243,8 +244,9 @@ func TestLiveFSCommandSurface(t *testing.T) { {"fs", "read-file", "help"}, {"fs", "chmod-file", "help"}, {"fs", "create-symlink", "help"}, {"fs", "create-hardlink", "help"}, {"fs", "create-layer", "help"}, {"fs", "list-layers", "help"}, + {"fs", "fork-layer", "help"}, {"fs", "list-layer-chain", "help"}, {"fs", "describe-layer", "help"}, {"fs", "diff-layer", "help"}, - {"fs", "create-layer-checkpoint", "help"}, {"fs", "rollback-layer", "help"}, + {"fs", "create-layer-checkpoint", "help"}, {"fs", "delete-layer", "help"}, {"fs", "rollback-layer", "help"}, {"fs", "commit-layer", "help"}, {"fs", "pack-file-system", "help"}, {"fs", "unpack-file-system", "help"}, {"fs", "drain-file-system", "help"}, {"fs", "cp", "help"}, {"fs", "cat", "help"}, {"fs", "ls", "help"}, @@ -276,6 +278,12 @@ func TestLiveFSCommandSurface(t *testing.T) { refreshDryRun := runTIWithInput(t, bin, "", []string{"TI_FS_TOKEN=" + drive9TestTokenWithVersion(selected.FSTenantID, 999), "TI_REGION_CODE=" + selected.FSPlacementRegionCode}, "--profile", profileName, "fs", "refresh-file-system-token", "--file-system-id", selected.FSTenantID, "--dry-run") refreshDryRun.wantExitCode(0) + forkDryRun := runTI(t, bin, "--profile", profileName, "fs", "fork-layer", "--parent-layer-ref", "layer-1", "--layer-name", "child", "--dry-run", "--query", "checks[].name") + forkDryRun.wantExitCode(0) + forkDryRun.wantStdoutContains("companion_capability") + deleteLayerDryRun := runTI(t, bin, "--profile", profileName, "fs", "delete-layer", "--layer-ref", "layer-1", "--cascade", "--dry-run", "--query", "checks[].name") + deleteLayerDryRun.wantExitCode(0) + deleteLayerDryRun.wantStdoutContains("companion_capability") unmountDryRun := runTI(t, bin, "--profile", profileName, "fs", "unmount-file-system", "--mount-path", "/tmp/ti-e2e-mount", "--ignore-absent", "--dry-run", "--query", "checks[].name") unmountDryRun.wantExitCode(0) for _, check := range []string{"input_validation", "mount_locator", "remote_mutation"} { @@ -302,7 +310,7 @@ func TestLiveFSCommandSurface(t *testing.T) { {"fs", "check-file-system"}, {"fs", "list-file-systems"}, {"fs", "describe-file-system"}, {"fs", "read-file"}, {"fs", "list-files"}, {"fs", "describe-file"}, {"fs", "search-file-content"}, {"fs", "find-files"}, {"fs", "list-layers"}, - {"fs", "describe-layer"}, {"fs", "diff-layer"}, + {"fs", "describe-layer"}, {"fs", "diff-layer"}, {"fs", "list-layer-chain", "--layer-ref", "layer-1"}, }) } @@ -945,6 +953,38 @@ func TestLiveFSDataPlaneLifecycle(t *testing.T) { createCheckpoint.wantExitCode(0) createCheckpoint.wantStdoutContains(checkpointID) + childID := layerID + "-child" + childDeleted := false + defer func() { + if childDeleted { + return + } + cleanup := runTI(t, bin, "--profile", profileName, "fs", "delete-layer", "--layer-ref", childID, "--cascade") + if cleanup.exitCode != 0 && cleanup.exitCode != 5 { + t.Logf("cleanup delete failed for child layer %s: exit=%d stdout=%s stderr=%s", childID, cleanup.exitCode, cleanup.stdout, cleanup.stderr) + } + }() + forkChild := runTI(t, bin, "--profile", profileName, "fs", "fork-layer", "--parent-layer-ref", layerID, "--layer-id", childID, "--layer-name", "live-e2e-child-"+suffix, "--checkpoint-id", checkpointID, "--actor-id", "ti-live-e2e-child") + forkChild.wantExitCode(0) + forkChild.wantStdoutContains(childID) + forkChild.wantStdoutContains(layerID) + + chain := runTI(t, bin, "--profile", profileName, "fs", "list-layer-chain", "--layer-ref", childID) + chain.wantExitCode(0) + chain.wantStdoutContains(`"layer_id": "` + layerID + `"`) + chain.wantStdoutContains(`"layer_id": "` + childID + `"`) + chainText := runTI(t, bin, "--profile", profileName, "fs", "list-layer-chain", "--layer-ref", childID, "--output", "text") + chainText.wantExitCode(0) + chainText.wantStdoutContains("PARENT_LAYER_ID") + + deleteChild := runTI(t, bin, "--profile", profileName, "fs", "delete-layer", "--layer-ref", childID) + deleteChild.wantExitCode(0) + deleteChild.wantStdoutContains(`"status": "abandoned"`) + childDeleted = true + describeDeletedChild := runTI(t, bin, "--profile", profileName, "fs", "describe-layer", "--layer-id", childID) + describeDeletedChild.wantExitCode(0) + describeDeletedChild.wantStdoutContains(`"state": "abandoned"`) + waitLiveFSResult(t, bin, []string{"--profile", profileName, "fs", "find-files", "--path", rootPath, "--file-name-pattern", "copy-layer.txt", "--layer-id", layerID, "--limit", "5"}, layerCopyPath, 2*time.Minute, "find file inside layer") commitLayer := runTI(t, bin, "--profile", profileName, "fs", "commit-layer", "--layer-id", layerID) @@ -2140,7 +2180,7 @@ func ensureLiveFSResource(t *testing.T, bin, profileName string) *config.Profile t.Fatalf("migrate live fs resource: %v", err) } requestedID := strings.TrimSpace(os.Getenv("TI_LIVE_FS_ID")) - list := runTI(t, bin, "--profile", profileName, "fs", "list-file-systems") + list := runTIWithInput(t, bin, "", []string{"TI_FS_FILE_SYSTEM_ID=", "TI_FS_TOKEN=", "TDC_FS_TOKEN="}, "--profile", profileName, "fs", "list-file-systems") list.wantExitCode(0) var inventory struct { RegionCode string `json:"region_code"` diff --git a/e2e/testdata/fake-drive9.go b/e2e/testdata/fake-drive9.go index 991c533..30a7acc 100644 --- a/e2e/testdata/fake-drive9.go +++ b/e2e/testdata/fake-drive9.go @@ -51,6 +51,30 @@ func main() { _ = file.Close() } args := os.Args[1:] + if hasPrefix(args, "fs", "layer", "help") { + fmt.Println("usage: drive9 fs layer ") + return + } + if hasPrefix(args, "mount", "--help") { + fmt.Println("-layer string") + fmt.Println("-checkpoint string") + return + } + if hasPrefix(args, "fs", "layer", "fork") { + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "layer_id": "child-id", "name": "child", "state": "active", "base_root_path": "/workspace", + "parent_layer_id": "parent-id", "origin_seq": 7, "origin_checkpoint_id": flagValue(args, "--checkpoint"), "root_layer_id": "root-id", "depth": 1, + }) + return + } + if hasPrefix(args, "fs", "layer", "chain") { + fmt.Println(`{"chain":[{"layer_id":"root-id","name":"root","state":"active","depth":0,"limit_seq":7,"base_root_path":"/workspace"},{"layer_id":"child-id","name":"child","state":"active","parent_layer_id":"root-id","origin_seq":7,"origin_checkpoint_id":"seed","depth":1,"limit_seq":7,"base_root_path":"/workspace"}]}`) + return + } + if hasPrefix(args, "fs", "layer", "delete") { + fmt.Println("ok") + return + } if hasPrefix(args, "create") { region := flagValue(args, "--region-code") id := "tenant-" + strings.ReplaceAll(region, "_", "-") @@ -115,7 +139,19 @@ func main() { os.Exit(1) } fmt.Println(`{"path":"/","size":0,"isdir":true}`) + return + } + if hasPrefix(args, "mount") { + fmt.Fprintln(os.Stderr, "drive9: mount mode: "+mountMode(args)) + return + } +} + +func mountMode(args []string) string { + if value := flagValue(args, "--mode"); value != "" { + return value } + return "webdav" } func hasPrefix(args []string, want ...string) bool { diff --git a/internal/api/fs/layer.go b/internal/api/fs/layer.go index 1d7195e..fce150c 100644 --- a/internal/api/fs/layer.go +++ b/internal/api/fs/layer.go @@ -43,17 +43,37 @@ type FSLayerCreateRequest struct { } type FSLayer struct { - LayerID string `json:"layer_id"` - BaseRootPath string `json:"base_root_path"` - Name string `json:"name"` - Tags map[string]string `json:"tags,omitempty"` - State string `json:"state"` - DurabilityMode string `json:"durability_mode"` - ActorID string `json:"actor_id"` - DurableSeq int64 `json:"durable_seq"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - SealedAt *time.Time `json:"sealed_at,omitempty"` + LayerID string `json:"layer_id"` + BaseRootPath string `json:"base_root_path"` + Name string `json:"name"` + Tags map[string]string `json:"tags,omitempty"` + State string `json:"state"` + DurabilityMode string `json:"durability_mode"` + ActorID string `json:"actor_id"` + DurableSeq int64 `json:"durable_seq"` + ParentLayerID string `json:"parent_layer_id,omitempty"` + OriginSeq int64 `json:"origin_seq,omitempty"` + OriginCheckpointID string `json:"origin_checkpoint_id,omitempty"` + RootLayerID string `json:"root_layer_id,omitempty"` + Depth int `json:"depth,omitempty"` + Origin string `json:"origin,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + SealedAt *time.Time `json:"sealed_at,omitempty"` +} + +type FSLayerChainFrame struct { + LayerID string `json:"layer_id"` + Name string `json:"name"` + State string `json:"state"` + ParentLayerID string `json:"parent_layer_id,omitempty"` + OriginSeq int64 `json:"origin_seq,omitempty"` + OriginCheckpointID string `json:"origin_checkpoint_id,omitempty"` + Depth int `json:"depth,omitempty"` + RootLayerID string `json:"root_layer_id,omitempty"` + BaseRootPath string `json:"base_root_path,omitempty"` + CreatedAt time.Time `json:"created_at"` + LimitSeq int64 `json:"limit_seq"` } type FSLayerEntry struct { diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 79049be..d21f6d4 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -882,9 +882,12 @@ func newFSCommand(info version.Info) *cobra.Command { newFSFindFilesCommand(info), newFSCreateLayerCommand(info), newFSListLayersCommand(info), + newFSForkLayerCommand(info), + newFSListLayerChainCommand(info), newFSDescribeLayerCommand(info), newFSDiffLayerCommand(info), newFSCreateLayerCheckpointCommand(info), + newFSDeleteLayerCommand(info), newFSRollbackLayerCommand(info), newFSCommitLayerCommand(info), newFSPackFileSystemCommand(info), @@ -1501,9 +1504,9 @@ func newFSCopyFileCommand(info version.Info) *cobra.Command { cmd.Flags().Bool("overwrite", false, "Replace an existing destination file.") cmd.Flags().Bool("create-parents", false, "Create missing local parent directories when copying from a TiDB Cloud file system.") cmd.Flags().Bool("append", false, "Append a local file content to a file in the TiDB Cloud file system.") - cmd.Flags().Bool("recursive", false, "Copy directory structure recursively.") + cmd.Flags().Bool("recursive", false, "Copy directory structure recursively; cannot be combined with --layer-id. Seed a layer through a writable FUSE mount instead.") cmd.Flags().Bool("resume", false, "Resume an active copy operation.") - cmd.Flags().String("layer-id", "", "Write the copied file content into a file system layer instead of the base file system.") + cmd.Flags().String("layer-id", "", "Write one copied file into a file system layer instead of the base file system; cannot be combined with --recursive.") cmd.Flags().StringArray("tag", nil, "Create tag(s) key=value for --to-remote operation; repeatable.") cmd.Flags().String("description", "", "The file description for --to-remote operation.") return cmd @@ -1946,6 +1949,80 @@ func newFSListLayersCommand(info version.Info) *cobra.Command { }, info) } +func newFSForkLayerCommand(info version.Info) *cobra.Command { + cmd := newControlPlaneCommand(controlPlaneCommandSpec{ + Use: "fork-layer", + Short: "Fork a copy-on-write child file system layer. (preview)", + Mutation: mutatingCommand, + Permission: authz.FSFileWrite, + Run: func(ctx commandContext) (any, error) { + service, profile, err := fsServiceAndProfile(ctx) + if err != nil { + return nil, err + } + opts, err := fsForkLayerOptions(ctx, profile) + if err != nil { + return nil, err + } + return service.ForkLayer(ctx.cmd.Context(), opts) + }, + DryRun: func(ctx commandContext) (dryrun.Result, error) { + service, profile, err := fsServiceAndProfile(ctx) + if err != nil { + return dryrun.Result{}, err + } + opts, err := fsForkLayerOptions(ctx, profile) + if err != nil { + return dryrun.Result{}, err + } + if err := tifs.ValidateLayerReference(opts.ParentLayerRef, "--parent-layer-ref"); err != nil { + return dryrun.Result{}, err + } + if opts.CheckpointID != "" { + if err := tifs.ValidateLayerReference(opts.CheckpointID, "--checkpoint-id"); err != nil { + return dryrun.Result{}, err + } + } + return service.DryRunLayerMutation(ctx.cmd.Context(), ctx.CommandPath(), "fork_layer", "POST", "/v1/layers/"+opts.ParentLayerRef+"/fork", map[string]any{ + "layer_id": opts.LayerID, + "name": opts.LayerName, + "checkpoint_id": opts.CheckpointID, + "actor_id": opts.ActorID, + }, profile, authz.FSFileWrite, tifs.LayerCapabilityFork) + }, + }, info) + cmd.Flags().String("parent-layer-ref", "", "The parent layer ID, unique name, or supported tag reference.") + cmd.Flags().String("layer-id", "", "The child layer ID. Normally it is generated by the service automatically.") + cmd.Flags().String("layer-name", "", "The name of the child layer.") + cmd.Flags().String("checkpoint-id", "", "Pin the child to this checkpoint of the parent layer.") + cmd.Flags().String("actor-id", "", "Actor ID identifying the child layer owner.") + markUsageRequired(cmd, "parent-layer-ref") + return cmd +} + +func newFSListLayerChainCommand(info version.Info) *cobra.Command { + cmd := newControlPlaneCommand(controlPlaneCommandSpec{ + Use: "list-layer-chain", + Short: "List a file system layer ancestry chain. (preview)", + Mutation: readOnlyCommand, + Permission: authz.FSFileRead, + Run: func(ctx commandContext) (any, error) { + service, profile, err := fsServiceAndProfile(ctx) + if err != nil { + return nil, err + } + layerRef, err := ctx.StringFlag("layer-ref") + if err != nil { + return nil, err + } + return service.ListLayerChain(ctx.cmd.Context(), tifs.ListLayerChainOptions{Profile: profile, LayerRef: layerRef}) + }, + }, info) + cmd.Flags().String("layer-ref", "", "The layer ID, unique name, or supported tag reference.") + markUsageRequired(cmd, "layer-ref") + return cmd +} + func newFSDescribeLayerCommand(info version.Info) *cobra.Command { cmd := newControlPlaneCommand(controlPlaneCommandSpec{ Use: "describe-layer", @@ -2032,6 +2109,44 @@ func newFSCreateLayerCheckpointCommand(info version.Info) *cobra.Command { return cmd } +func newFSDeleteLayerCommand(info version.Info) *cobra.Command { + cmd := newControlPlaneCommand(controlPlaneCommandSpec{ + Use: "delete-layer", + Short: "Abandon a file system layer. (preview)", + Mutation: mutatingCommand, + Permission: authz.FSFileWrite, + Run: func(ctx commandContext) (any, error) { + service, profile, err := fsServiceAndProfile(ctx) + if err != nil { + return nil, err + } + opts, err := fsDeleteLayerOptions(ctx, profile) + if err != nil { + return nil, err + } + return service.DeleteLayer(ctx.cmd.Context(), opts) + }, + DryRun: func(ctx commandContext) (dryrun.Result, error) { + service, profile, err := fsServiceAndProfile(ctx) + if err != nil { + return dryrun.Result{}, err + } + opts, err := fsDeleteLayerOptions(ctx, profile) + if err != nil { + return dryrun.Result{}, err + } + if err := tifs.ValidateLayerReference(opts.LayerRef, "--layer-ref"); err != nil { + return dryrun.Result{}, err + } + return service.DryRunLayerMutation(ctx.cmd.Context(), ctx.CommandPath(), "delete_layer", "DELETE", fmt.Sprintf("/v1/layers/%s?cascade=%t", opts.LayerRef, opts.Cascade), nil, profile, authz.FSFileWrite, tifs.LayerCapabilityDelete) + }, + }, info) + cmd.Flags().String("layer-ref", "", "The layer ID, unique name, or supported tag reference.") + cmd.Flags().Bool("cascade", false, "Abandon live descendants before abandoning this layer.") + markUsageRequired(cmd, "layer-ref") + return cmd +} + func newFSRollbackLayerCommand(info version.Info) *cobra.Command { cmd := newControlPlaneCommand(controlPlaneCommandSpec{ Use: "rollback-layer", @@ -2216,12 +2331,14 @@ func newFSMountFileSystemCommand(info version.Info) *cobra.Command { cmd.Flags().Int64("read-cache-size-mb", 128, "FUSE read cache size in MiB. 0 uses the default.") cmd.Flags().Int64("read-cache-max-file-mb", 4, "Maximum file size admitted to the FUSE read cache in MiB. 0 uses the default.") cmd.Flags().Duration("read-cache-ttl", 30*time.Second, "FUSE read cache Time-to-Live.") - cmd.Flags().Bool("write-back-cache", true, "Persist FUSE writes locally before writing them to the file system on flush.") + cmd.Flags().Bool("write-back-cache", true, "Persist FUSE writes locally before writing them to the file system on flush; unavailable for checkpoint mounts.") cmd.Flags().String("mount-profile", "", "Mount profile: coding-agent, portable, or none. Default: none.") cmd.Flags().String("local-root", "", "Local overlay root. Default: ~/.ti/local/fs/.") cmd.Flags().StringArray("pack-path", nil, "Local overlay path included by automatic or manual pack. Repeatable.") cmd.Flags().String("unpack-archive-path", "", "Restore the pack archive before mounting.") cmd.Flags().Bool("no-auto-unpack", false, "Skip default auto-unpack for portable mount profile before mounting.") + cmd.Flags().String("layer-ref", "", "Mount through this writable layer ID, unique name, or supported tag reference; requires FUSE.") + cmd.Flags().String("checkpoint-id", "", "Mount this checkpoint of --layer-ref read-only; requires FUSE.") markUsageRequired(cmd, "mount-path") return cmd } @@ -2635,6 +2752,42 @@ func fsCreateLayerCheckpointOptions(ctx commandContext, profile *config.Profile) }, nil } +func fsForkLayerOptions(ctx commandContext, profile *config.Profile) (tifs.ForkLayerOptions, error) { + parentLayerRef, err := ctx.StringFlag("parent-layer-ref") + if err != nil { + return tifs.ForkLayerOptions{}, err + } + layerID, err := ctx.StringFlag("layer-id") + if err != nil { + return tifs.ForkLayerOptions{}, err + } + layerName, err := ctx.StringFlag("layer-name") + if err != nil { + return tifs.ForkLayerOptions{}, err + } + checkpointID, err := ctx.StringFlag("checkpoint-id") + if err != nil { + return tifs.ForkLayerOptions{}, err + } + actorID, err := ctx.StringFlag("actor-id") + if err != nil { + return tifs.ForkLayerOptions{}, err + } + return tifs.ForkLayerOptions{Profile: profile, ParentLayerRef: parentLayerRef, LayerID: layerID, LayerName: layerName, CheckpointID: checkpointID, ActorID: actorID}, nil +} + +func fsDeleteLayerOptions(ctx commandContext, profile *config.Profile) (tifs.DeleteLayerOptions, error) { + layerRef, err := ctx.StringFlag("layer-ref") + if err != nil { + return tifs.DeleteLayerOptions{}, err + } + cascade, err := ctx.BoolFlag("cascade") + if err != nil { + return tifs.DeleteLayerOptions{}, err + } + return tifs.DeleteLayerOptions{Profile: profile, LayerRef: layerRef, Cascade: cascade}, nil +} + func fsMountOptions(ctx commandContext, profile *config.Profile) (tifs.MountFileSystemOptions, error) { fileSystemID, err := ctx.StringFlag("file-system-id") if err != nil { @@ -2700,6 +2853,14 @@ func fsMountOptions(ctx commandContext, profile *config.Profile) (tifs.MountFile if err != nil { return tifs.MountFileSystemOptions{}, err } + layerRef, err := ctx.StringFlag("layer-ref") + if err != nil { + return tifs.MountFileSystemOptions{}, err + } + checkpointID, err := ctx.StringFlag("checkpoint-id") + if err != nil { + return tifs.MountFileSystemOptions{}, err + } return tifs.MountFileSystemOptions{ Profile: profile, FileSystemName: fileSystemID, @@ -2707,17 +2868,21 @@ func fsMountOptions(ctx commandContext, profile *config.Profile) (tifs.MountFile RemotePath: remotePath, Driver: driver, ReadOnly: readOnly, + ReadOnlySet: ctx.FlagChanged("read-only"), ReadyTimeout: readyTimeout, CacheDir: cacheDir, ReadCacheMB: readCacheMB, ReadCacheFileMB: readCacheFileMB, ReadCacheTTL: readCacheTTL, WriteBackCache: writeBackCache, + WriteBackCacheSet: ctx.FlagChanged("write-back-cache"), MountProfile: mountProfile, LocalRoot: localRoot, PackPaths: packPaths, UnpackArchivePath: unpackArchivePath, NoAutoUnpack: noAutoUnpack, + LayerRef: layerRef, + CheckpointID: checkpointID, }, nil } diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index 30788d0..949f496 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -50,6 +50,14 @@ type drive9CommandResult struct { Stdout string `json:"stdout,omitempty"` } +const ( + drive9CapabilityLayerFork = "layer-fork" + drive9CapabilityLayerChain = "layer-chain" + drive9CapabilityLayerDelete = "layer-delete" + drive9CapabilityLayerMount = "layer-mount" + drive9CapabilityCheckpointMount = "checkpoint-mount" +) + func (s Service) drive9Runner() fswrap.Runner { return fswrap.Runner{ HomeDir: s.HomeDir, @@ -72,6 +80,77 @@ func (s Service) drive9Run(ctx context.Context, profile *config.Profile, args [] }) } +func (s Service) requireDrive9Capabilities(ctx context.Context, profile *config.Profile, capabilities ...string) error { + needLayerHelp := false + needMountHelp := false + for _, capability := range capabilities { + switch capability { + case drive9CapabilityLayerFork, drive9CapabilityLayerChain, drive9CapabilityLayerDelete: + needLayerHelp = true + case drive9CapabilityLayerMount, drive9CapabilityCheckpointMount: + needMountHelp = true + default: + return apperr.New("fs.companion_capability_unknown", "runtime", 1, fmt.Sprintf("unknown ti fs companion capability %q", capability)) + } + } + + layerHelp := "" + if needLayerHelp { + result, err := s.drive9Runner().Run(ctx, fswrap.RunOptions{Profile: profile, Args: []string{"fs", "layer", "help"}, CaptureStdout: true}) + if err != nil { + return incompatibleDrive9Error(err) + } + layerHelp = string(result.Stdout) + "\n" + string(result.Stderr) + } + mountHelp := "" + if needMountHelp { + result, err := s.drive9Runner().Run(ctx, fswrap.RunOptions{Profile: profile, Args: []string{"mount", "--help"}, CaptureStdout: true}) + if err != nil { + return incompatibleDrive9Error(err) + } + mountHelp = string(result.Stdout) + "\n" + string(result.Stderr) + } + + for _, capability := range capabilities { + available := false + switch capability { + case drive9CapabilityLayerFork: + available = layerHelpHasCommand(layerHelp, "fork") + case drive9CapabilityLayerChain: + available = layerHelpHasCommand(layerHelp, "chain") + case drive9CapabilityLayerDelete: + available = layerHelpHasCommand(layerHelp, "delete") + case drive9CapabilityLayerMount: + available = strings.Contains(mountHelp, "-layer string") || strings.Contains(mountHelp, "--layer string") + case drive9CapabilityCheckpointMount: + available = strings.Contains(mountHelp, "-checkpoint string") || strings.Contains(mountHelp, "--checkpoint string") + } + if !available { + return incompatibleDrive9Error(nil) + } + } + return nil +} + +func layerHelpHasCommand(help, command string) bool { + for _, line := range strings.FieldsFunc(help, func(r rune) bool { + return r == '<' || r == '>' || r == '|' || r == ' ' || r == '\t' || r == '\r' || r == '\n' + }) { + if line == command { + return true + } + } + return false +} + +func incompatibleDrive9Error(cause error) error { + message := "the installed ti-drive9 companion does not support filesystem layer fork workflows; update or reinstall ti after a compatible companion release is available" + if cause != nil { + return apperr.Wrap("fs.companion_incompatible", "runtime", 1, message, cause) + } + return apperr.New("fs.companion_incompatible", "runtime", 1, message) +} + func (s Service) drive9RunTransientRetry(ctx context.Context, profile *config.Profile, args []string, capture bool) (fswrap.Result, error) { var lastResult fswrap.Result var lastErr error @@ -597,6 +676,57 @@ func (s Service) drive9ListLayers(ctx context.Context, opts ListLayersOptions) ( return out, nil } +func (s Service) drive9ForkLayer(ctx context.Context, opts ForkLayerOptions) (LayerResult, error) { + if err := s.requireDrive9Capabilities(ctx, opts.Profile, drive9CapabilityLayerFork); err != nil { + return LayerResult{}, err + } + args := []string{"fs", "layer", "fork", "--json"} + appendFlagValue(&args, "--id", opts.LayerID) + appendFlagValue(&args, "--name", opts.LayerName) + appendFlagValue(&args, "--checkpoint", opts.CheckpointID) + appendFlagValue(&args, "--actor", opts.ActorID) + args = append(args, strings.TrimSpace(opts.ParentLayerRef)) + result, err := s.drive9Run(ctx, opts.Profile, args, true) + if err != nil { + return LayerResult{}, err + } + var layer apifs.FSLayer + if err := json.Unmarshal(result.Stdout, &layer); err != nil { + return LayerResult{}, apperr.Wrap("fs.companion_decode", "runtime", 1, "decode ti fs layer fork response", err) + } + return LayerResult{FSLayer: layer}, nil +} + +func (s Service) drive9ListLayerChain(ctx context.Context, opts ListLayerChainOptions) (LayerChainResult, error) { + if err := s.requireDrive9Capabilities(ctx, opts.Profile, drive9CapabilityLayerChain); err != nil { + return LayerChainResult{}, err + } + result, err := s.drive9RunTransientRetry(ctx, opts.Profile, []string{"fs", "layer", "chain", "--json", strings.TrimSpace(opts.LayerRef)}, true) + if err != nil { + return LayerChainResult{}, err + } + var out LayerChainResult + if err := json.Unmarshal(result.Stdout, &out); err != nil { + return LayerChainResult{}, apperr.Wrap("fs.companion_decode", "runtime", 1, "decode ti fs layer chain response", err) + } + return out, nil +} + +func (s Service) drive9DeleteLayer(ctx context.Context, opts DeleteLayerOptions) (DeleteLayerResult, error) { + if err := s.requireDrive9Capabilities(ctx, opts.Profile, drive9CapabilityLayerDelete); err != nil { + return DeleteLayerResult{}, err + } + args := []string{"fs", "layer", "delete"} + if opts.Cascade { + args = append(args, "--cascade") + } + args = append(args, strings.TrimSpace(opts.LayerRef)) + if _, err := s.drive9Run(ctx, opts.Profile, args, true); err != nil { + return DeleteLayerResult{}, err + } + return DeleteLayerResult{Operation: "delete_layer", LayerRef: strings.TrimSpace(opts.LayerRef), Status: "abandoned", Cascade: opts.Cascade}, nil +} + func (s Service) drive9DescribeLayer(ctx context.Context, opts DescribeLayerOptions) (LayerResult, error) { result, err := s.drive9RunTransientRetry(ctx, opts.Profile, []string{"fs", "layer", "status", "--json", strings.TrimSpace(opts.LayerID)}, true) if err != nil { @@ -1122,6 +1252,17 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem if opts.ReadOnly { args = append(args, "--read-only") } + if opts.LayerRef != "" { + capabilities := []string{drive9CapabilityLayerMount} + if opts.CheckpointID != "" { + capabilities = append(capabilities, drive9CapabilityCheckpointMount) + } + if err := s.requireDrive9Capabilities(ctx, opts.Profile, capabilities...); err != nil { + return MountResult{}, err + } + appendFlagValue(&args, "--layer", opts.LayerRef) + appendFlagValue(&args, "--checkpoint", opts.CheckpointID) + } fuseRequested := strings.TrimSpace(opts.Driver) == "fuse" if fuseRequested { appendFlagValue(&args, "--cache-dir", opts.CacheDir) @@ -1151,7 +1292,7 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem if err != nil { return MountResult{}, err } - if err := s.writeDrive9MountLocator(opts.Profile, opts.MountPath, "fs"); err != nil { + if err := s.writeDrive9MountLocator(opts.Profile, opts.MountPath, "fs", opts); err != nil { _, _ = s.drive9Run(ctx, opts.Profile, []string{"umount", opts.MountPath}, false) return MountResult{}, err } @@ -1160,7 +1301,7 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem if driver == "" { driver = "auto" } - return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: opts.Profile.FSResourceName, MountPath: opts.MountPath, RemotePath: remotePath, Driver: driver, Endpoint: &endpoint, MountProfile: opts.MountProfile, LocalRoot: opts.LocalRoot, PackPaths: opts.PackPaths, WriteBackCache: opts.WriteBackCache}, nil + return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: opts.Profile.FSResourceName, MountPath: opts.MountPath, RemotePath: remotePath, Driver: driver, Endpoint: &endpoint, MountProfile: opts.MountProfile, LocalRoot: opts.LocalRoot, PackPaths: opts.PackPaths, WriteBackCache: opts.WriteBackCache, LayerRef: opts.LayerRef, CheckpointID: opts.CheckpointID, ReadOnly: opts.ReadOnly}, nil } func (s Service) drive9DrainFileSystem(ctx context.Context, opts DrainFileSystemOptions) (DrainResult, error) { @@ -1221,7 +1362,7 @@ func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSy return UnmountResult{Status: "unmounted", MountPath: opts.MountPath}, nil } -func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kind string) error { +func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kind string, mountOpts ...MountFileSystemOptions) error { if profile == nil { return apperr.New("fs.missing_profile", "config", 2, "active profile is required") } @@ -1238,6 +1379,10 @@ func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kin return apperr.Wrap("fs.write_mount_locator", "runtime", 1, "construct ti fs mount locator", err) } locator = locator.WithTokenCorrelation(profile.FSTenantID, profile.FSTokenID, fsTokenFingerprint(profile.FSAPIKey)) + if len(mountOpts) > 0 { + opts := mountOpts[0] + locator = locator.WithLayerView(opts.RemotePath, opts.LayerRef, opts.CheckpointID, opts.ReadOnly) + } if _, err := mountlocator.Write(homeDir, locator); err != nil { return apperr.Wrap("fs.write_mount_locator", "runtime", 1, "write ti fs mount locator", err) } @@ -1283,6 +1428,9 @@ func (s Service) drive9MountLocatorProfile(base *config.Profile, mountPath strin } func drive9CopyArgs(opts CopyFileOptions) ([]string, string, string, error) { + if opts.Recursive && strings.TrimSpace(opts.LayerID) != "" { + return nil, "", "", apperr.New("fs.recursive_layer_copy_unsupported", "usage", 2, "--recursive cannot be combined with --layer-id; mount the layer with --driver fuse and copy the directory through the mounted path") + } args := []string{"fs", "cp"} if opts.Resume { args = append(args, "--resume") diff --git a/internal/fs/drive9_companion_test.go b/internal/fs/drive9_companion_test.go index 67ed677..5d9eb45 100644 --- a/internal/fs/drive9_companion_test.go +++ b/internal/fs/drive9_companion_test.go @@ -310,6 +310,186 @@ func TestDrive9CopyDoesNotRetryNonReplayableStreamsOrAppend(t *testing.T) { } } +func TestDrive9LayerForkChainAndDeleteTranslateToCompanion(t *testing.T) { + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(t.TempDir(), companion) + profile := dataProfile() + + forked, err := service.ForkLayer(context.Background(), ForkLayerOptions{ + Profile: profile, ParentLayerRef: "research-base", LayerID: "child-id", LayerName: "style-analyst", CheckpointID: "seed", ActorID: "hermes", + }) + if err != nil { + t.Fatalf("ForkLayer failed: %v", err) + } + if forked.LayerID != "child-id" || forked.ParentLayerID != "parent-id" || forked.OriginCheckpointID != "seed" || forked.Depth != 1 { + t.Fatalf("unexpected fork response: %#v", forked) + } + forkCall := requireFakeDrive9Call(t, recordPath, "fs", "layer", "fork") + wantFork := []string{"fs", "layer", "fork", "--json", "--id", "child-id", "--name", "style-analyst", "--checkpoint", "seed", "--actor", "hermes", "research-base"} + if fmt.Sprint(forkCall.Args) != fmt.Sprint(wantFork) { + t.Fatalf("fork args = %#v, want %#v", forkCall.Args, wantFork) + } + + chain, err := service.ListLayerChain(context.Background(), ListLayerChainOptions{Profile: profile, LayerRef: "style-analyst"}) + if err != nil { + t.Fatalf("ListLayerChain failed: %v", err) + } + if len(chain.Chain) != 2 || chain.Chain[0].LayerID != "root-id" || chain.Chain[1].LimitSeq != 12 { + t.Fatalf("unexpected chain response: %#v", chain) + } + chainCall := requireFakeDrive9Call(t, recordPath, "fs", "layer", "chain") + wantChain := []string{"fs", "layer", "chain", "--json", "style-analyst"} + if fmt.Sprint(chainCall.Args) != fmt.Sprint(wantChain) { + t.Fatalf("chain args = %#v, want %#v", chainCall.Args, wantChain) + } + + deleted, err := service.DeleteLayer(context.Background(), DeleteLayerOptions{Profile: profile, LayerRef: "style-brief", Cascade: true}) + if err != nil { + t.Fatalf("DeleteLayer failed: %v", err) + } + if deleted.Status != "abandoned" || !deleted.Cascade || deleted.LayerRef != "style-brief" { + t.Fatalf("unexpected delete response: %#v", deleted) + } + deleteCall := requireFakeDrive9Call(t, recordPath, "fs", "layer", "delete") + wantDelete := []string{"fs", "layer", "delete", "--cascade", "style-brief"} + if fmt.Sprint(deleteCall.Args) != fmt.Sprint(wantDelete) { + t.Fatalf("delete args = %#v, want %#v", deleteCall.Args, wantDelete) + } +} + +func TestDrive9LayerTipForkAndLeafDeleteOmitOptionalFlags(t *testing.T) { + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(t.TempDir(), companion) + profile := dataProfile() + + forked, err := service.ForkLayer(context.Background(), ForkLayerOptions{Profile: profile, ParentLayerRef: "parent-id"}) + if err != nil { + t.Fatalf("ForkLayer failed: %v", err) + } + if forked.LayerID == "" { + t.Fatalf("fork response did not include a generated child ID: %#v", forked) + } + forkCall := requireFakeDrive9Call(t, recordPath, "fs", "layer", "fork") + wantFork := []string{"fs", "layer", "fork", "--json", "parent-id"} + if fmt.Sprint(forkCall.Args) != fmt.Sprint(wantFork) { + t.Fatalf("tip fork args = %#v, want %#v", forkCall.Args, wantFork) + } + + deleted, err := service.DeleteLayer(context.Background(), DeleteLayerOptions{Profile: profile, LayerRef: "leaf-id"}) + if err != nil { + t.Fatalf("DeleteLayer failed: %v", err) + } + if deleted.Cascade { + t.Fatalf("leaf delete unexpectedly enabled cascade: %#v", deleted) + } + deleteCall := requireFakeDrive9Call(t, recordPath, "fs", "layer", "delete") + wantDelete := []string{"fs", "layer", "delete", "leaf-id"} + if fmt.Sprint(deleteCall.Args) != fmt.Sprint(wantDelete) { + t.Fatalf("leaf delete args = %#v, want %#v", deleteCall.Args, wantDelete) + } +} + +func TestDrive9LayerWorkflowRejectsIncompatibleCompanion(t *testing.T) { + companion, _ := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_OLD_LAYER", "1") + _, err := testCompanionService(t.TempDir(), companion).ForkLayer(context.Background(), ForkLayerOptions{Profile: dataProfile(), ParentLayerRef: "parent"}) + if apperr.CodeFor(err) != "fs.companion_incompatible" { + t.Fatalf("error = %v, want fs.companion_incompatible", err) + } +} + +func TestDrive9DeleteLayerPreservesDescendantConflict(t *testing.T) { + companion, _ := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_DELETE_DESCENDANT", "1") + _, err := testCompanionService(t.TempDir(), companion).DeleteLayer(context.Background(), DeleteLayerOptions{Profile: dataProfile(), LayerRef: "parent"}) + if err == nil || !strings.Contains(apperr.MessageFor(err), "live descendants") { + t.Fatalf("delete descendant conflict was not preserved: %v", err) + } +} + +func TestDrive9LayerCheckpointMountIsReadOnlyAndRecorded(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + mountPath := filepath.Join(t.TempDir(), "checkpoint") + result, err := testCompanionService(home, companion).MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, RemotePath: "/research/q3-market", Driver: "fuse", LayerRef: "style-analyst", CheckpointID: "v5", + }) + if err != nil { + t.Fatalf("MountFileSystem failed: %v", err) + } + if !result.ReadOnly || result.LayerRef != "style-analyst" || result.CheckpointID != "v5" || result.Driver != "fuse" { + t.Fatalf("unexpected layer mount result: %#v", result) + } + mountCall := requireFakeDrive9CallWithoutArg(t, recordPath, "--help", "mount") + want := []string{"mount", "--mode", "fuse", "--read-only", "--layer", "style-analyst", "--checkpoint", "v5", ":/research/q3-market", mountPath} + if fmt.Sprint(mountCall.Args) != fmt.Sprint(want) { + t.Fatalf("mount args = %#v, want %#v", mountCall.Args, want) + } + locator, _, err := mountlocator.Read(home, mountPath) + if err != nil { + t.Fatalf("read mount locator: %v", err) + } + if locator.LayerRef != "style-analyst" || locator.CheckpointID != "v5" || !locator.ReadOnly || locator.RemotePath != "/research/q3-market" { + t.Fatalf("unexpected layer mount locator: %#v", locator) + } +} + +func TestLayerMountValidationRejectsUnsupportedInputs(t *testing.T) { + profile := dataProfile() + service := Service{} + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{Profile: profile, MountPath: "/tmp/x", Driver: "fuse", CheckpointID: "v5"}); apperr.CodeFor(err) != "fs.checkpoint_requires_layer" { + t.Fatalf("checkpoint-only error = %v", err) + } + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{Profile: profile, MountPath: "/tmp/x", Driver: "webdav", LayerRef: "layer"}); apperr.CodeFor(err) != "fs.layer_mount_requires_fuse" { + t.Fatalf("webdav layer error = %v", err) + } + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{Profile: profile, MountPath: "/tmp/x", Driver: "fuse", LayerRef: "layer", CheckpointID: "v5", ReadOnlySet: true}); apperr.CodeFor(err) != "fs.checkpoint_mount_read_only" { + t.Fatalf("explicit writable checkpoint error = %v", err) + } + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{Profile: profile, MountPath: "/tmp/x", Driver: "fuse", LayerRef: "layer", CheckpointID: "v5", WriteBackCache: true, WriteBackCacheSet: true}); apperr.CodeFor(err) != "fs.checkpoint_mount_read_only" { + t.Fatalf("checkpoint write-back error = %v", err) + } +} + +func TestLayerReferenceValidation(t *testing.T) { + for _, tc := range []struct { + name string + value string + code string + }{ + {name: "empty", value: " ", code: "fs.missing_layer_reference"}, + {name: "slash", value: "parent/child", code: "fs.invalid_layer_reference"}, + {name: "backslash", value: `parent\child`, code: "fs.invalid_layer_reference"}, + {name: "control", value: "parent\nchild", code: "fs.invalid_layer_reference"}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := ValidateLayerReference(tc.value, "--layer-ref"); apperr.CodeFor(err) != tc.code { + t.Fatalf("error = %v, want %s", err, tc.code) + } + }) + } + if err := ValidateLayerReference("tag:run=123", "--layer-ref"); err != nil { + t.Fatalf("valid tag reference failed: %v", err) + } +} + +func TestDrive9RecursiveLayerCopyIsRejectedLocally(t *testing.T) { + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + _, err := testCompanionService(t.TempDir(), companion).CopyFile(context.Background(), CopyFileOptions{ + Profile: dataProfile(), FromLocal: ".", ToRemote: "/research/q3-market", Recursive: true, LayerID: "research-base", + }) + if apperr.CodeFor(err) != "fs.recursive_layer_copy_unsupported" { + t.Fatalf("error = %v, want fs.recursive_layer_copy_unsupported", err) + } + if _, statErr := os.Stat(recordPath); !os.IsNotExist(statErr) { + t.Fatalf("rejected copy invoked companion: %v", statErr) + } +} + func TestDrive9MissingCompanionIsActionable(t *testing.T) { t.Setenv("PATH", t.TempDir()) _, err := Service{CompanionPath: filepath.Join(t.TempDir(), "missing-ti-drive9")}.ReadFile(context.Background(), ReadFileOptions{ @@ -633,6 +813,35 @@ func main() { return } switch { + case len(args) >= 3 && args[0] == "fs" && args[1] == "layer" && args[2] == "help": + if os.Getenv("TI_FAKE_DRIVE9_OLD_LAYER") == "1" { + fmt.Println("usage: drive9 fs layer ") + } else { + fmt.Println("usage: drive9 fs layer ") + } + case len(args) >= 2 && args[0] == "mount" && args[1] == "--help": + if os.Getenv("TI_FAKE_DRIVE9_OLD_LAYER") == "1" { + fmt.Println("usage: drive9 mount [flags] [:/remote] ") + } else { + fmt.Println("-layer string") + fmt.Println("-checkpoint string") + } + case len(args) >= 3 && args[0] == "fs" && args[1] == "layer" && args[2] == "fork": + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "layer_id": "child-id", "name": "style-analyst", "state": "active", "base_root_path": "/research/q3-market", + "parent_layer_id": "parent-id", "origin_seq": 12, "origin_checkpoint_id": "seed", "root_layer_id": "root-id", "depth": 1, + }) + case len(args) >= 3 && args[0] == "fs" && args[1] == "layer" && args[2] == "chain": + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"chain": []map[string]any{ + {"layer_id": "root-id", "name": "research-base", "state": "active", "depth": 0, "limit_seq": 12, "base_root_path": "/research/q3-market"}, + {"layer_id": "child-id", "name": "style-analyst", "state": "active", "depth": 1, "parent_layer_id": "root-id", "origin_seq": 12, "limit_seq": 12, "origin_checkpoint_id": "seed", "base_root_path": "/research/q3-market"}, + }}) + case len(args) >= 3 && args[0] == "fs" && args[1] == "layer" && args[2] == "delete": + if os.Getenv("TI_FAKE_DRIVE9_DELETE_DESCENDANT") == "1" { + fmt.Fprintln(os.Stderr, "fs layer delete: layer has live descendants") + os.Exit(1) + } + fmt.Fprintln(os.Stdout, "ok") case args[0] == "create": if path := os.Getenv("TI_FAKE_DRIVE9_BREAK_CREDENTIAL_ROOT"); path != "" { _ = os.RemoveAll(path) @@ -648,7 +857,11 @@ func main() { }) case args[0] == "mount" && (len(args) == 1 || args[1] != "drain"): fmt.Fprintln(os.Stderr, "component: drive9 mount") - fmt.Fprintln(os.Stderr, "drive9: mount mode: webdav") + if flagValue(args, "--mode") == "fuse" { + fmt.Fprintln(os.Stderr, "drive9: mount mode: fuse") + } else { + fmt.Fprintln(os.Stderr, "drive9: mount mode: webdav") + } if os.Getenv("TI_FAKE_DRIVE9_MOUNT_FAIL") == "1" { fmt.Fprintln(os.Stderr, "mount: drive9 mount: background mount exited before becoming ready") os.Exit(1) @@ -828,6 +1041,17 @@ func requireFakeDrive9Call(t *testing.T, recordPath string, prefix ...string) fa return fakeDrive9Call{} } +func requireFakeDrive9CallWithoutArg(t *testing.T, recordPath, excluded string, prefix ...string) fakeDrive9Call { + t.Helper() + for _, call := range readFakeDrive9Calls(t, recordPath) { + if hasArgPrefix(call.Args, prefix) && !containsArg(call.Args, excluded) { + return call + } + } + t.Fatalf("missing fake companion call with prefix %#v without %q", prefix, excluded) + return fakeDrive9Call{} +} + func hasArgPrefix(args, prefix []string) bool { if len(args) < len(prefix) { return false diff --git a/internal/fs/fscred/credential.go b/internal/fs/fscred/credential.go index 81f496f..de565af 100644 --- a/internal/fs/fscred/credential.go +++ b/internal/fs/fscred/credential.go @@ -324,6 +324,12 @@ func ListCredentials(homeDir, profileName string) ([]Credential, error) { } credential, err := GetCredential(homeDir, profileName, id) if err != nil { + if apperr.CodeFor(err) == "fs.credential_not_found" { + contents, readErr := os.ReadDir(filepath.Join(dir, entry.Name())) + if readErr == nil && len(contents) == 0 { + continue + } + } return nil, err } credentials = append(credentials, credential) diff --git a/internal/fs/fscred/credential_test.go b/internal/fs/fscred/credential_test.go index 0b9c917..fa0345a 100644 --- a/internal/fs/fscred/credential_test.go +++ b/internal/fs/fscred/credential_test.go @@ -338,6 +338,25 @@ func TestMigrateNameRegistryMultipleResourcesAliasesAndIdempotency(t *testing.T) } } +func TestListCredentialsIgnoresEmptyPreflightDirectory(t *testing.T) { + home := t.TempDir() + profile := credentialTestProfile() + if _, err := StoreCredential(home, profile, "tenant-ready", "aws-us-east-1", wrappedToken(t, "tenant-ready"), false); err != nil { + t.Fatal(err) + } + if err := PrepareCredentialTarget(home, profile.Name, "tenant-preflight"); err != nil { + t.Fatal(err) + } + + credentials, err := ListCredentials(home, profile.Name) + if err != nil { + t.Fatal(err) + } + if len(credentials) != 1 || credentials[0].FileSystemID != "tenant-ready" { + t.Fatalf("credentials = %#v, want only tenant-ready", credentials) + } +} + func TestMigrateNameRegistryDoesNotRestoreDeletedCredential(t *testing.T) { home := t.TempDir() profile := credentialTestProfile() diff --git a/internal/fs/layer.go b/internal/fs/layer.go index 70ff688..71344bb 100644 --- a/internal/fs/layer.go +++ b/internal/fs/layer.go @@ -8,6 +8,7 @@ import ( "sort" "strings" "text/tabwriter" + "unicode" apifs "github.com/tidbcloud/ti-cli/internal/api/fs" "github.com/tidbcloud/ti-cli/internal/apperr" @@ -17,6 +18,11 @@ import ( "github.com/tidbcloud/ti-cli/internal/fs/fscred" ) +const ( + LayerCapabilityFork = drive9CapabilityLayerFork + LayerCapabilityDelete = drive9CapabilityLayerDelete +) + type CreateLayerOptions struct { Profile *config.Profile LayerID string @@ -36,6 +42,26 @@ type ListLayersOptions struct { Profile *config.Profile } +type ForkLayerOptions struct { + Profile *config.Profile + ParentLayerRef string + LayerID string + LayerName string + CheckpointID string + ActorID string +} + +type ListLayerChainOptions struct { + Profile *config.Profile + LayerRef string +} + +type DeleteLayerOptions struct { + Profile *config.Profile + LayerRef string + Cascade bool +} + type LayerEntriesOptions struct { Profile *config.Profile LayerID string @@ -119,6 +145,17 @@ type LayerListResult struct { Layers []apifs.FSLayer `json:"layers"` } +type LayerChainResult struct { + Chain []apifs.FSLayerChainFrame `json:"chain"` +} + +type DeleteLayerResult struct { + Operation string `json:"operation"` + LayerRef string `json:"layer_ref"` + Status string `json:"status"` + Cascade bool `json:"cascade"` +} + type LayerEntriesResult struct { LayerID string `json:"layer_id"` Entries []apifs.FSLayerEntry `json:"entries"` @@ -155,6 +192,32 @@ func (s Service) ListLayers(ctx context.Context, opts ListLayersOptions) (LayerL return s.drive9ListLayers(ctx, opts) } +func (s Service) ForkLayer(ctx context.Context, opts ForkLayerOptions) (LayerResult, error) { + if err := validateLayerReference(opts.ParentLayerRef, "--parent-layer-ref"); err != nil { + return LayerResult{}, err + } + if strings.TrimSpace(opts.CheckpointID) != "" { + if err := validateLayerReference(opts.CheckpointID, "--checkpoint-id"); err != nil { + return LayerResult{}, err + } + } + return s.drive9ForkLayer(ctx, opts) +} + +func (s Service) ListLayerChain(ctx context.Context, opts ListLayerChainOptions) (LayerChainResult, error) { + if err := validateLayerReference(opts.LayerRef, "--layer-ref"); err != nil { + return LayerChainResult{}, err + } + return s.drive9ListLayerChain(ctx, opts) +} + +func (s Service) DeleteLayer(ctx context.Context, opts DeleteLayerOptions) (DeleteLayerResult, error) { + if err := validateLayerReference(opts.LayerRef, "--layer-ref"); err != nil { + return DeleteLayerResult{}, err + } + return s.drive9DeleteLayer(ctx, opts) +} + func (s Service) DescribeLayer(ctx context.Context, opts DescribeLayerOptions) (LayerResult, error) { return s.drive9DescribeLayer(ctx, opts) } @@ -283,7 +346,7 @@ func (s Service) CommitLayer(ctx context.Context, opts LayerActionOptions) (Laye return s.drive9CommitLayer(ctx, opts) } -func (s Service) DryRunLayerMutation(ctx context.Context, commandPath, operation, method, requestPath string, body any, profile *config.Profile, permission authz.Permission) (dryrun.Result, error) { +func (s Service) DryRunLayerMutation(ctx context.Context, commandPath, operation, method, requestPath string, body any, profile *config.Profile, permission authz.Permission, capabilities ...string) (dryrun.Result, error) { if profile == nil { return dryrun.Result{}, apperr.New("fs.missing_profile", "config", 2, "active profile is required") } @@ -301,6 +364,12 @@ func (s Service) DryRunLayerMutation(ctx context.Context, commandPath, operation } else { return dryrun.Result{}, apperr.New("auth.missing_fs_api_key", "authentication", 3, fmt.Sprintf("authentication required: missing fs_api_key for profile %q. Create or configure a ti fs resource first.", profileName(profile))) } + if len(capabilities) > 0 { + if err := s.requireDrive9Capabilities(ctx, profile, capabilities...); err != nil { + return dryrun.Result{}, err + } + checks = append(checks, dryrun.Check{Name: "companion_capability", Status: "passed", Message: strings.Join(capabilities, ", ")}) + } return dryrun.New( commandPath, operation, @@ -313,6 +382,26 @@ func (s Service) DryRunLayerMutation(ctx context.Context, commandPath, operation ), nil } +func ValidateLayerReference(value, flagName string) error { + return validateLayerReference(value, flagName) +} + +func validateLayerReference(value, flagName string) error { + value = strings.TrimSpace(value) + if value == "" { + return apperr.New("fs.missing_layer_reference", "usage", 2, flagName+" is required") + } + if strings.ContainsAny(value, "/\\") { + return apperr.New("fs.invalid_layer_reference", "usage", 2, fmt.Sprintf("%s must not contain path separators", flagName)) + } + for _, r := range value { + if unicode.IsControl(r) { + return apperr.New("fs.invalid_layer_reference", "usage", 2, fmt.Sprintf("%s must not contain control characters", flagName)) + } + } + return nil +} + func layerEntryRequest(opts CreateLayerEntryOptions) (apifs.FSLayerEntryRequest, error) { remotePath, err := normalizeRemotePath(opts.Path) if err != nil { @@ -448,6 +537,22 @@ func (r LayerResult) Human() string { if r.ActorID != "" { lines = append(lines, "Actor: "+r.ActorID) } + if r.ParentLayerID != "" { + lines = append(lines, + "Parent layer ID: "+r.ParentLayerID, + fmt.Sprintf("Origin seq: %d", r.OriginSeq), + fmt.Sprintf("Depth: %d", r.Depth), + ) + } + if r.OriginCheckpointID != "" { + lines = append(lines, "Origin checkpoint ID: "+r.OriginCheckpointID) + } + if r.RootLayerID != "" { + lines = append(lines, "Root layer ID: "+r.RootLayerID) + } + if r.Origin != "" { + lines = append(lines, "Origin: "+r.Origin) + } if r.DurableSeq != 0 { lines = append(lines, fmt.Sprintf("Durable seq: %d", r.DurableSeq)) } @@ -457,6 +562,21 @@ func (r LayerResult) Human() string { return strings.Join(lines, "\n") } +func (r LayerChainResult) Human() string { + var out strings.Builder + writer := tabwriter.NewWriter(&out, 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintln(writer, "LAYER_ID\tNAME\tSTATE\tDEPTH\tPARENT_LAYER_ID\tORIGIN_SEQ\tLIMIT_SEQ\tORIGIN_CHECKPOINT_ID\tBASE_ROOT") + for _, frame := range r.Chain { + _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%d\t%s\t%d\t%d\t%s\t%s\n", frame.LayerID, frame.Name, frame.State, frame.Depth, frame.ParentLayerID, frame.OriginSeq, frame.LimitSeq, frame.OriginCheckpointID, frame.BaseRootPath) + } + _ = writer.Flush() + return strings.TrimRight(out.String(), "\n") +} + +func (r DeleteLayerResult) Human() string { + return fmt.Sprintf("%s layer=%s status=%s cascade=%t", r.Operation, r.LayerRef, r.Status, r.Cascade) +} + func (r LayerListResult) Human() string { var out strings.Builder writer := tabwriter.NewWriter(&out, 0, 0, 2, ' ', 0) diff --git a/internal/fs/mount.go b/internal/fs/mount.go index d48e2c7..7576c8b 100644 --- a/internal/fs/mount.go +++ b/internal/fs/mount.go @@ -42,17 +42,21 @@ type MountFileSystemOptions struct { RemotePath string Driver string ReadOnly bool + ReadOnlySet bool ReadyTimeout time.Duration CacheDir string ReadCacheMB int64 ReadCacheFileMB int64 ReadCacheTTL time.Duration WriteBackCache bool + WriteBackCacheSet bool MountProfile string LocalRoot string PackPaths []string UnpackArchivePath string NoAutoUnpack bool + LayerRef string + CheckpointID string } type UnmountFileSystemOptions struct { @@ -90,6 +94,9 @@ type MountResult struct { MountProfile string `json:"mount_profile,omitempty"` LocalRoot string `json:"local_root,omitempty"` PackPaths []string `json:"pack_paths,omitempty"` + LayerRef string `json:"layer_ref,omitempty"` + CheckpointID string `json:"checkpoint_id,omitempty"` + ReadOnly bool `json:"read_only,omitempty"` } type MountRemoteSnapshot struct { @@ -136,10 +143,29 @@ type MountRuntimeCheck struct { } func (s Service) MountFileSystem(ctx context.Context, opts MountFileSystemOptions) (MountResult, error) { + var err error + opts, err = normalizeLayerMountOptions(opts) + if err != nil { + return MountResult{}, err + } return s.drive9MountFileSystem(ctx, opts) } func (s Service) DryRunMountFileSystem(ctx context.Context, commandPath string, opts MountFileSystemOptions) (dryrun.Result, error) { + var err error + opts, err = normalizeLayerMountOptions(opts) + if err != nil { + return dryrun.Result{}, err + } + if opts.LayerRef != "" { + capabilities := []string{drive9CapabilityLayerMount} + if opts.CheckpointID != "" { + capabilities = append(capabilities, drive9CapabilityCheckpointMount) + } + if err := s.requireDrive9Capabilities(ctx, opts.Profile, capabilities...); err != nil { + return dryrun.Result{}, err + } + } inputs, err := s.mountInputs(opts) if err != nil { return dryrun.Result{}, err @@ -155,6 +181,12 @@ func (s Service) DryRunMountFileSystem(ctx context.Context, commandPath string, } else { checks = append(checks, dryrun.Check{Name: "mount_driver", Status: "passed", Message: inputs.driver.Name()}) } + if opts.LayerRef != "" { + checks = append(checks, dryrun.Check{Name: "layer_view", Status: "passed", Message: opts.LayerRef}) + } + if opts.CheckpointID != "" { + checks = append(checks, dryrun.Check{Name: "checkpoint_view", Status: "passed", Message: opts.CheckpointID + " (read-only)"}) + } description := "normal execution starts a local ti fs FUSE runtime and mounts it at the requested path" if inputs.driver.Name() == "webdav" { description = "normal execution starts a local WebDAV bridge and mounts it at the requested path" @@ -166,11 +198,70 @@ func (s Service) DryRunMountFileSystem(ctx context.Context, commandPath string, Description: description, Method: "GET", Path: "/v1/status", + Body: map[string]any{ + "driver": opts.Driver, + "layer_ref": opts.LayerRef, + "checkpoint_id": opts.CheckpointID, + "read_only": opts.ReadOnly, + }, }, checks..., ), nil } +func normalizeLayerMountOptions(opts MountFileSystemOptions) (MountFileSystemOptions, error) { + opts.LayerRef = strings.TrimSpace(opts.LayerRef) + opts.CheckpointID = strings.TrimSpace(opts.CheckpointID) + if opts.CheckpointID != "" && opts.LayerRef == "" { + return MountFileSystemOptions{}, apperr.New("fs.checkpoint_requires_layer", "usage", 2, "--checkpoint-id requires --layer-ref") + } + if opts.LayerRef == "" { + return opts, nil + } + if err := validateLayerReference(opts.LayerRef, "--layer-ref"); err != nil { + return MountFileSystemOptions{}, err + } + if opts.CheckpointID != "" { + if err := validateLayerReference(opts.CheckpointID, "--checkpoint-id"); err != nil { + return MountFileSystemOptions{}, err + } + if opts.ReadOnlySet && !opts.ReadOnly { + return MountFileSystemOptions{}, apperr.New("fs.checkpoint_mount_read_only", "usage", 2, "checkpoint mounts are read-only; remove --read-only=false") + } + if opts.WriteBackCacheSet && opts.WriteBackCache { + return MountFileSystemOptions{}, apperr.New("fs.checkpoint_mount_read_only", "usage", 2, "checkpoint mounts are read-only; remove --write-back-cache=true") + } + opts.ReadOnly = true + opts.WriteBackCache = false + } + driverName := strings.TrimSpace(opts.Driver) + if driverName == "" { + driverName = "auto" + } + if driverName == "webdav" { + return MountFileSystemOptions{}, apperr.New("fs.layer_mount_requires_fuse", "usage", 2, "layer and checkpoint mounts require FUSE; install or enable FUSE and use --driver fuse") + } + if driverName == "auto" { + driver, err := mountdriver.Resolve("auto") + if err != nil { + return MountFileSystemOptions{}, apperr.New("fs.invalid_mount_driver", "usage", 2, err.Error()) + } + if driver.Name() != "fuse" { + return MountFileSystemOptions{}, apperr.New("fs.layer_mount_requires_fuse", "usage", 2, "automatic mount selection would use WebDAV, but layer and checkpoint mounts require FUSE; install or enable FUSE and use --driver fuse") + } + if err := driver.CheckPrerequisites(); err != nil { + return MountFileSystemOptions{}, apperr.New("fs.layer_mount_requires_fuse", "usage", 2, "layer and checkpoint mounts require FUSE: "+err.Error()) + } + opts.Driver = "fuse" + return opts, nil + } + if driverName != "fuse" { + return MountFileSystemOptions{}, apperr.New("fs.invalid_mount_driver", "usage", 2, fmt.Sprintf("unsupported ti fs mount driver %q; supported values: auto, fuse, webdav", driverName)) + } + opts.Driver = "fuse" + return opts, nil +} + func (s Service) UnmountFileSystem(ctx context.Context, opts UnmountFileSystemOptions) (UnmountResult, error) { return s.drive9UnmountFileSystem(ctx, opts) } @@ -692,6 +783,15 @@ func (r MountResult) Human() string { "Remote path: " + r.RemotePath, "Driver: " + r.Driver, } + if r.LayerRef != "" { + lines = append(lines, "Layer: "+r.LayerRef) + } + if r.CheckpointID != "" { + lines = append(lines, "Checkpoint: "+r.CheckpointID) + } + if r.ReadOnly { + lines = append(lines, "Read only: true") + } if r.PID > 0 { lines = append(lines, fmt.Sprintf("PID: %d", r.PID)) } diff --git a/internal/fs/mountlocator/locator.go b/internal/fs/mountlocator/locator.go index 703b33f..ebabb61 100644 --- a/internal/fs/mountlocator/locator.go +++ b/internal/fs/mountlocator/locator.go @@ -23,6 +23,18 @@ type Locator struct { FileSystemID string `json:"file_system_id,omitempty"` TokenID string `json:"token_id,omitempty"` TokenFingerprint string `json:"token_fingerprint,omitempty"` + RemotePath string `json:"remote_path,omitempty"` + LayerRef string `json:"layer_ref,omitempty"` + CheckpointID string `json:"checkpoint_id,omitempty"` + ReadOnly bool `json:"read_only,omitempty"` +} + +func (l Locator) WithLayerView(remotePath, layerRef, checkpointID string, readOnly bool) Locator { + l.RemotePath = strings.TrimSpace(remotePath) + l.LayerRef = strings.TrimSpace(layerRef) + l.CheckpointID = strings.TrimSpace(checkpointID) + l.ReadOnly = readOnly + return l } func New(profile, fileSystemName, regionCode, companionHome, mountPath, kind string) (Locator, error) { diff --git a/internal/update/update.go b/internal/update/update.go index 018846a..1e825e3 100644 --- a/internal/update/update.go +++ b/internal/update/update.go @@ -742,9 +742,79 @@ func (c client) downloadDrive9Companion(ctx context.Context, baseURL string) (do cleanup() return downloadedBinary{}, func() {}, apperr.Wrap("update.extract_artifact", "runtime", 1, "make ti-drive9 executable", err) } + if err := validateDrive9Companion(ctx, tempPath); err != nil { + cleanup() + return downloadedBinary{}, func() {}, err + } return downloadedBinary{Path: tempPath, SHA256: expectedSHA}, cleanup, nil } +func validateDrive9Companion(ctx context.Context, path string) error { + layerHelp, err := drive9HelpOutput(ctx, path, "fs", "layer", "help") + if err != nil { + return incompatibleDrive9CompanionError("read filesystem layer command surface", layerHelp, err) + } + for _, command := range []string{"fork", "chain", "delete"} { + if !drive9HelpHasToken(layerHelp, command) { + return incompatibleDrive9CompanionError("missing `drive9 fs layer "+command+"`", layerHelp, nil) + } + } + + mountHelp, err := drive9HelpOutput(ctx, path, "mount", "--help") + if err != nil { + return incompatibleDrive9CompanionError("read filesystem mount command surface", mountHelp, err) + } + for _, flag := range []string{"layer", "checkpoint"} { + if !drive9HelpHasFlag(mountHelp, flag) { + return incompatibleDrive9CompanionError("missing `drive9 mount --"+flag+"`", mountHelp, nil) + } + } + return nil +} + +func drive9HelpOutput(ctx context.Context, path string, args ...string) (string, error) { + validateCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + cmd := exec.CommandContext(validateCtx, path, args...) + output, err := cmd.CombinedOutput() + return strings.TrimSpace(string(output)), err +} + +func drive9HelpHasToken(help, token string) bool { + return slicesContains(strings.FieldsFunc(help, func(r rune) bool { + return r == '<' || r == '>' || r == '|' || r == ' ' || r == '\t' || r == '\r' || r == '\n' + }), token) +} + +func drive9HelpHasFlag(help, flag string) bool { + for _, field := range strings.Fields(help) { + if strings.TrimLeft(field, "-") == flag { + return true + } + } + return false +} + +func slicesContains(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func incompatibleDrive9CompanionError(detail, output string, cause error) error { + message := "downloaded ti-drive9 companion is incompatible with this ti release: " + detail + "; retry after the matching companion release is published" + if output != "" { + message += ": " + output + } + if cause != nil { + return apperr.Wrap("update.companion_incompatible", "runtime", 1, message, cause) + } + return apperr.New("update.companion_incompatible", "runtime", 1, message) +} + func drive9ArtifactName(goos, goarch string) (string, error) { if goos == "" { goos = runtime.GOOS diff --git a/internal/update/update_test.go b/internal/update/update_test.go index 8bbf271..c6dd926 100644 --- a/internal/update/update_test.go +++ b/internal/update/update_test.go @@ -147,7 +147,7 @@ func TestApplyReplacesOwnedUnixBinary(t *testing.T) { tag: "v0.2.0", assets: map[string][]byte{ artifactForRuntime(t): archiveBytes, - companionArtifact: []byte("#!/bin/sh\necho companion\n"), + companionArtifact: []byte(compatibleDrive9CompanionScript), }, }) current := filepath.Join(t.TempDir(), "ti") @@ -187,6 +187,75 @@ func TestApplyReplacesOwnedUnixBinary(t *testing.T) { } } +func TestApplyRejectsIncompatibleCompanionBeforeReplacingFiles(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows self-update is unsupported") + } + + archiveBytes := tarGzBinary(t, "ti", "#!/bin/sh\necho 'ti 0.2.0 (test, now, linux/amd64)'\n") + companionArtifact, err := drive9ArtifactName(runtime.GOOS, runtime.GOARCH) + if err != nil { + t.Fatal(err) + } + server := fakeReleaseServer(t, releaseFixture{ + tag: "v0.2.0", + assets: map[string][]byte{ + artifactForRuntime(t): archiveBytes, + companionArtifact: []byte("#!/bin/sh\necho old-companion\n"), + }, + }) + installDir := t.TempDir() + current := filepath.Join(installDir, "ti") + companion := filepath.Join(installDir, companionBinaryName()) + if err := os.WriteFile(current, []byte("#!/bin/sh\necho current\n"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(companion, []byte("existing companion"), 0o755); err != nil { + t.Fatal(err) + } + + _, err = Apply(context.Background(), version.Info{ + Version: "0.1.0", + OS: runtime.GOOS, + Arch: runtime.GOARCH, + InstallSource: "archive", + }, ApplyOptions{ + ReleaseAPIBaseURL: server.URL, + Drive9ReleaseURL: server.URL + "/assets", + ExecutablePath: current, + }) + if err == nil { + t.Fatal("expected incompatible companion rejection") + } + if got := apperr.CodeFor(err); got != "update.companion_incompatible" { + t.Fatalf("error code = %q, want update.companion_incompatible: %v", got, err) + } + currentBytes, readErr := os.ReadFile(current) + if readErr != nil { + t.Fatal(readErr) + } + companionBytes, readErr := os.ReadFile(companion) + if readErr != nil { + t.Fatal(readErr) + } + if string(currentBytes) != "#!/bin/sh\necho current\n" || string(companionBytes) != "existing companion" { + t.Fatalf("targets changed after incompatible companion rejection: ti=%q companion=%q", currentBytes, companionBytes) + } +} + +const compatibleDrive9CompanionScript = `#!/bin/sh +if [ "$1 $2 $3" = "fs layer help" ]; then + echo 'usage: drive9 fs layer ' + exit 0 +fi +if [ "$1 $2" = "mount --help" ]; then + echo ' -layer string' + echo ' -checkpoint string' + exit 0 +fi +echo companion +` + func TestApplyRefusesProtectedTargetWithoutChangingFiles(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Windows self-update is unsupported") diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 9babc20..47a5227 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -22,6 +22,28 @@ function Warn($Message) { Write-Warning $Message } +function Assert-CompanionCommandSurface($CompanionPath) { + $layerHelp = (& $CompanionPath fs layer help 2>&1 | Out-String) + if ($LASTEXITCODE -ne 0) { + Fail "downloaded ti-drive9 companion cannot report its filesystem layer commands" + } + foreach ($command in @("fork", "chain", "delete")) { + if ($layerHelp -notmatch "(?m)(^|[\s<|])$command([\s>|]|$)") { + Fail "downloaded ti-drive9 companion is incompatible: missing drive9 fs layer $command" + } + } + + $mountHelp = (& $CompanionPath mount --help 2>&1 | Out-String) + if ($LASTEXITCODE -ne 0) { + Fail "downloaded ti-drive9 companion cannot report its mount options" + } + foreach ($flag in @("layer", "checkpoint")) { + if ($mountHelp -notmatch "(?m)(^|\s)--?$flag(\s|$)") { + Fail "downloaded ti-drive9 companion is incompatible: missing drive9 mount --$flag" + } + } +} + function Resolve-InstallDir { if (-not [string]::IsNullOrWhiteSpace($env:TI_INSTALL_DIR) -and -not [string]::IsNullOrWhiteSpace($env:TDC_INSTALL_DIR) -and @@ -220,6 +242,8 @@ try { Fail "checksum mismatch for $CompanionArtifact" } + Assert-CompanionCommandSurface $CompanionPath + Expand-Archive -Path $ArchivePath -DestinationPath $TempDir.FullName -Force $Extracted = Get-ChildItem -Path $TempDir.FullName -Recurse -Filter "ti.exe" | Select-Object -First 1 if (-not $Extracted) { diff --git a/scripts/install.sh b/scripts/install.sh index 4e843ee..1029f9f 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -101,6 +101,27 @@ download() { fi } +validate_companion() { + companion_path="$1" + if ! layer_help="$("$companion_path" fs layer help 2>&1)"; then + error "downloaded ti-drive9 companion cannot report its filesystem layer commands" + fi + for command in fork chain delete; do + if ! printf '%s\n' "$layer_help" | grep -Eq "(^|[[:space:]<|])${command}([[:space:]>|]|$)"; then + error "downloaded ti-drive9 companion is incompatible: missing drive9 fs layer ${command}" + fi + done + + if ! mount_help="$("$companion_path" mount --help 2>&1)"; then + error "downloaded ti-drive9 companion cannot report its mount options" + fi + for flag in layer checkpoint; do + if ! printf '%s\n' "$mount_help" | grep -Eq "(^|[[:space:]])--?${flag}([[:space:]]|$)"; then + error "downloaded ti-drive9 companion is incompatible: missing drive9 mount --${flag}" + fi + done +} + case "$(uname -s)" in Darwin) OS="darwin" ;; Linux) OS="linux" ;; @@ -359,6 +380,7 @@ if [ -z "$FOUND" ]; then fi chmod 0755 "$FOUND" chmod 0755 "${TMP_DIR}/${COMPANION_ARTIFACT}" +validate_companion "${TMP_DIR}/${COMPANION_ARTIFACT}" run_home_migration "$FOUND" install_file "$FOUND" "$TARGET" install_file "${TMP_DIR}/${COMPANION_ARTIFACT}" "$COMPANION_TARGET"