From d96549b7ed6c79e5fdf13fa036cd637d7853cd53 Mon Sep 17 00:00:00 2001 From: "victor.linroth.sensmetry" Date: Mon, 5 Jan 2026 14:30:41 +0100 Subject: [PATCH 1/4] Adds a resolved manifest to `sysand_env` keeping track of what is part of the current project. Signed-off-by: victor.linroth.sensmetry --- core/src/commands/sources.rs | 7 +- core/src/env/local_directory.rs | 150 +++++++++++++++++++++++++++- core/src/lock.rs | 29 ++++-- sysand/src/commands/add.rs | 1 + sysand/src/commands/clone.rs | 1 + sysand/src/commands/env.rs | 2 + sysand/src/commands/sync.rs | 17 +++- sysand/src/lib.rs | 1 + sysand/tests/cli_env.rs | 15 ++- sysand/tests/cli_include_exclude.rs | 4 +- sysand/tests/cli_sync.rs | 84 +++++++++++++++- 11 files changed, 286 insertions(+), 25 deletions(-) diff --git a/core/src/commands/sources.rs b/core/src/commands/sources.rs index 00a1499d..76f5eaf9 100644 --- a/core/src/commands/sources.rs +++ b/core/src/commands/sources.rs @@ -137,5 +137,10 @@ pub fn enumerate_projects_lock( Vec<::InterchangeProjectRead>, ResolutionError<::ReadError>, > { - lock.resolve_projects(env) + let projects = lock + .resolve_projects(env)? + .into_iter() + .filter_map(|(_, project_read)| project_read) + .collect(); + Ok(projects) } diff --git a/core/src/env/local_directory.rs b/core/src/env/local_directory.rs index 22303ae9..2982bbac 100644 --- a/core/src/env/local_directory.rs +++ b/core/src/env/local_directory.rs @@ -1,16 +1,24 @@ // SPDX-FileCopyrightText: © 2025 Sysand contributors // SPDX-License-Identifier: MIT OR Apache-2.0 -use camino::{Utf8Path, Utf8PathBuf}; -use camino_tempfile::NamedUtf8TempFile; -use sha2::Sha256; use std::{ + collections::HashMap, + fmt::Display, fs, io::{self, BufRead, BufReader, Read, Write}, + num::TryFromIntError, }; +use camino::{Utf8Path, Utf8PathBuf}; +use camino_tempfile::NamedUtf8TempFile; +use sha2::Sha256; +use thiserror::Error; +use toml_edit::{Array, ArrayOfTables, DocumentMut, Item, Table, Value, value}; + use crate::{ + commands::sources::{LocalSourcesError, do_sources_local_src_project_no_deps}, env::{PutProjectError, ReadEnvironment, WriteEnvironment, segment_uri_generic}, + lock::{Lock, ResolutionError, Source, multiline_list}, project::{ local_src::{LocalSrcError, LocalSrcProject, PathError}, utils::{ @@ -19,8 +27,6 @@ use crate::{ }, }; -use thiserror::Error; - #[derive(Clone, Debug)] pub struct LocalDirectoryEnvironment { pub environment_path: Utf8PathBuf, @@ -28,6 +34,8 @@ pub struct LocalDirectoryEnvironment { pub const DEFAULT_ENV_NAME: &str = "sysand_env"; +pub const DEFAULT_MANIFEST_NAME: &str = "current.toml"; + pub const ENTRIES_PATH: &str = "entries.txt"; pub const VERSIONS_PATH: &str = "versions.txt"; @@ -617,3 +625,135 @@ impl WriteEnvironment for LocalDirectoryEnvironment { Ok(()) } } + +#[derive(Debug, Error)] +pub enum ResolvedManifestError { + #[error(transparent)] + ResolutionError(#[from] ResolutionError), + #[error("too many dependencies, unable to convert to i64: {0}")] + TooManyDependencies(TryFromIntError), + #[error(transparent)] + LocalSources(#[from] LocalSourcesError), + #[error(transparent)] + Canonicalization(#[from] Box), +} + +impl Lock { + pub fn to_resolved_manifest>( + &self, + env: &LocalDirectoryEnvironment, + root_path: P, + ) -> Result { + let resolved_projects = self.resolve_projects(env)?; + + let indices = resolved_projects + .iter() + .map(|(p, _)| p) + .enumerate() + .flat_map(|(num, p)| p.identifiers.iter().map(move |iri| (iri.clone(), num))) + .map(|(iri, num)| i64::try_from(num).map(|num| (iri, num))) + .collect::, _>>() + .map_err(ResolvedManifestError::TooManyDependencies)?; + let indices = HashMap::::from_iter(indices); + + let mut projects = vec![]; + for (project, storage) in resolved_projects { + let usages = project + .usages + .iter() + .filter_map(|usage| indices.get(&usage.resource)) + .copied() + .collect(); + + if let Some(storage) = storage { + let directory = storage.root_path(); + projects.push(ResolvedProject { + name: project.name, + location: ResolvedLocation::Directory(directory), + usages, + }); + } else if let [Source::Editable { editable }, ..] = project.sources.as_slice() { + let project_path = root_path.as_ref().join(editable.as_str()); + let editable_project = LocalSrcProject { + project_path: wrapfs::canonicalize(project_path)?, + }; + let files = do_sources_local_src_project_no_deps(&editable_project, true)? + .into_iter() + .collect(); + projects.push(ResolvedProject { + name: project.name, + location: ResolvedLocation::Files(files), + usages, + }); + } + } + + Ok(ResolvedManifest { projects }) + } +} + +#[derive(Debug)] +pub struct ResolvedManifest { + pub projects: Vec, +} + +impl Display for ResolvedManifest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.to_toml()) + } +} + +impl ResolvedManifest { + pub fn to_toml(&self) -> DocumentMut { + let mut doc = DocumentMut::new(); + let mut projects = ArrayOfTables::new(); + for project in &self.projects { + projects.push(project.to_toml()); + } + doc.insert("project", Item::ArrayOfTables(projects)); + + doc + } +} + +#[derive(Debug)] +pub enum ResolvedLocation { + Directory(Utf8PathBuf), + Files(Vec), +} + +#[derive(Debug)] +pub struct ResolvedProject { + pub name: Option, + pub location: ResolvedLocation, + pub usages: Vec, +} + +impl ResolvedProject { + pub fn to_toml(&self) -> Table { + let mut table = Table::new(); + if let Some(name) = &self.name { + table.insert("name", value(name)); + } + match &self.location { + ResolvedLocation::Directory(dir) => { + table.insert("directory", value(dir.as_str())); + } + ResolvedLocation::Files(files) => { + if !files.is_empty() { + table.insert( + "files", + value(multiline_list( + files.iter().map(|f| Value::from(f.as_str())), + )), + ); + } + } + } + if !self.usages.is_empty() { + let usages = Array::from_iter(self.usages.iter().copied().map(Value::from)); + table.insert("usages", value(usages)); + } + table + } +} diff --git a/core/src/lock.rs b/core/src/lock.rs index 40086982..e84b81cb 100644 --- a/core/src/lock.rs +++ b/core/src/lock.rs @@ -175,18 +175,33 @@ pub enum ValidationError { }, } +pub type ProjectResolution = ( + Project, + Option<::InterchangeProjectRead>, +); + impl Lock { pub fn resolve_projects( &self, env: &Env, - ) -> Result< - Vec<::InterchangeProjectRead>, - ResolutionError, - > { + ) -> Result>, ResolutionError> { let mut missing = vec![]; let mut found = vec![]; for project in &self.projects { + // Projects without sources (default for standard libraries) and + // projects with editable sources won't be installed in environment. + match project.sources.as_slice() { + [] => { + continue; + } + [Source::Editable { editable: _ }, ..] => { + found.push((project.clone(), None)); + continue; + } + _ => {} + } + let checksum = &project.checksum; let mut resolved_project = None; @@ -206,8 +221,8 @@ impl Lock { } } - if let Some(success) = resolved_project { - found.push(success); + if resolved_project.is_some() { + found.push((project.clone(), resolved_project)); } else { missing.push(project.clone()); } @@ -560,7 +575,7 @@ impl From for Usage { } } -fn multiline_list(elements: impl Iterator>) -> Array { +pub fn multiline_list(elements: impl Iterator>) -> Array { let mut array: Array = elements .map(|item| { let mut value = item.into(); diff --git a/sysand/src/commands/add.rs b/sysand/src/commands/add.rs index 35564386..28cffbad 100644 --- a/sysand/src/commands/add.rs +++ b/sysand/src/commands/add.rs @@ -63,6 +63,7 @@ pub fn command_add, Policy: HTTPAuthentication>( command_sync( &lock, project_root, + true, &mut env, client, &provided_iris, diff --git a/sysand/src/commands/clone.rs b/sysand/src/commands/clone.rs index ad907785..07c445b9 100644 --- a/sysand/src/commands/clone.rs +++ b/sysand/src/commands/clone.rs @@ -215,6 +215,7 @@ pub fn command_clone( command_sync( &lock, &project.inner().project_path, + true, &mut env, client, &provided_iris, diff --git a/sysand/src/commands/env.rs b/sysand/src/commands/env.rs index 25eab94e..d743c940 100644 --- a/sysand/src/commands/env.rs +++ b/sysand/src/commands/env.rs @@ -138,6 +138,7 @@ pub fn command_env_install( command_sync( &lock, project_root, + false, &mut env, client, &provided_iris, @@ -254,6 +255,7 @@ pub fn command_env_install_path, Policy: HTTPAuthentication>( command_sync( &lock, project_root, + false, &mut env, client, &provided_iris, diff --git a/sysand/src/commands/sync.rs b/sysand/src/commands/sync.rs index 5753d62a..eac76e0f 100644 --- a/sysand/src/commands/sync.rs +++ b/sysand/src/commands/sync.rs @@ -9,18 +9,21 @@ use url::ParseError; use sysand_core::{ auth::HTTPAuthentication, - env::local_directory::LocalDirectoryEnvironment, + env::local_directory::{DEFAULT_ENV_NAME, DEFAULT_MANIFEST_NAME, LocalDirectoryEnvironment}, lock::Lock, project::{ AsSyncProjectTokio, ProjectReadAsync, local_kpar::LocalKParProject, local_src::LocalSrcProject, memory::InMemoryProject, reqwest_kpar_download::ReqwestKparDownloadedProject, reqwest_src::ReqwestSrcProjectAsync, + utils::wrapfs, }, }; +#[allow(clippy::too_many_arguments)] pub fn command_sync, Policy: HTTPAuthentication>( lock: &Lock, project_root: P, + update_manifest: bool, env: &mut LocalDirectoryEnvironment, client: reqwest_middleware::ClientWithMiddleware, provided_iris: &HashMap>, @@ -57,5 +60,17 @@ pub fn command_sync, Policy: HTTPAuthentication>( ), provided_iris, )?; + + if update_manifest { + let manifest = lock.to_resolved_manifest(env, &project_root)?; + wrapfs::write( + project_root + .as_ref() + .join(DEFAULT_ENV_NAME) + .join(DEFAULT_MANIFEST_NAME), + manifest.to_string(), + )?; + } + Ok(()) } diff --git a/sysand/src/lib.rs b/sysand/src/lib.rs index 89e6f55b..049f36b5 100644 --- a/sysand/src/lib.rs +++ b/sysand/src/lib.rs @@ -376,6 +376,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { command_sync( &lock, project_root, + true, &mut local_environment, client, &provided_iris, diff --git a/sysand/tests/cli_env.rs b/sysand/tests/cli_env.rs index a602661a..702880b7 100644 --- a/sysand/tests/cli_env.rs +++ b/sysand/tests/cli_env.rs @@ -7,7 +7,7 @@ use assert_cmd::prelude::*; use camino::Utf8Path; use mockito::Server; use predicates::prelude::*; -use sysand_core::env::local_directory::DEFAULT_ENV_NAME; +use sysand_core::env::local_directory::{DEFAULT_ENV_NAME, ENTRIES_PATH, VERSIONS_PATH}; // pub due to https://github.com/rust-lang/rust/issues/46379 mod common; @@ -31,13 +31,12 @@ fn env_init_empty_env() -> Result<(), Box> { if path.is_dir() { assert_eq!(path.strip_prefix(&cwd)?, env_path); } else { - // if path.is_file() - assert_eq!(path.strip_prefix(&cwd)?, env_path.join("entries.txt")); + assert_eq!(path.strip_prefix(&cwd)?, env_path.join(ENTRIES_PATH)); } } assert_eq!( - std::fs::File::open(cwd.join("sysand_env/entries.txt"))? + std::fs::File::open(cwd.join(DEFAULT_ENV_NAME).join(ENTRIES_PATH))? .metadata()? .len(), 0 @@ -75,7 +74,7 @@ fn env_install_from_local_dir() -> Result<(), Box> { .stderr(predicate::str::contains("`urn:kpar:test` 0.0.1")); assert_eq!( - std::fs::read_to_string(cwd.join(env_path).join("entries.txt"))?, + std::fs::read_to_string(cwd.join(env_path).join(ENTRIES_PATH))?, "urn:kpar:test\n" ); @@ -84,7 +83,7 @@ fn env_install_from_local_dir() -> Result<(), Box> { assert!(cwd.join(env_path).join(test_hash).is_dir()); assert_eq!( - std::fs::read_to_string(cwd.join(env_path).join(test_hash).join("versions.txt"))?, + std::fs::read_to_string(cwd.join(env_path).join(test_hash).join(VERSIONS_PATH))?, "0.0.1\n" ); @@ -127,7 +126,7 @@ fn env_install_from_local_dir() -> Result<(), Box> { assert_eq!(entries.len(), 1); - assert_eq!(entries[0].file_name(), "entries.txt"); + assert_eq!(entries[0].file_name(), ENTRIES_PATH); assert_eq!(std::fs::read_to_string(entries[0].path())?, ""); @@ -190,7 +189,7 @@ fn env_install_from_http_kpar() -> Result<(), Box> { out.assert().success(); assert_eq!( - std::fs::read_to_string(cwd.join(env_path).join("entries.txt"))?, + std::fs::read_to_string(cwd.join(env_path).join(ENTRIES_PATH))?, format!("{}\n", &project_url) ); diff --git a/sysand/tests/cli_include_exclude.rs b/sysand/tests/cli_include_exclude.rs index 98f3567e..a6174d1d 100644 --- a/sysand/tests/cli_include_exclude.rs +++ b/sysand/tests/cli_include_exclude.rs @@ -27,13 +27,13 @@ fn include_and_exclude_simple() -> Result<(), Box> { None, )?; + out.assert().success(); + { let mut sysml_file = std::fs::File::create(cwd.join("test.sysml"))?; sysml_file.write_all(b"package P;\n")?; } - out.assert().success(); - let out = run_sysand_in(&cwd, ["include", "test.sysml", "--compute-checksum"], None)?; out.assert().success(); diff --git a/sysand/tests/cli_sync.rs b/sysand/tests/cli_sync.rs index 946c135b..fc9c4eae 100644 --- a/sysand/tests/cli_sync.rs +++ b/sysand/tests/cli_sync.rs @@ -7,12 +7,68 @@ use assert_cmd::prelude::*; use indexmap::IndexMap; use mockito::Matcher; use predicates::prelude::*; -use sysand_core::commands::lock::DEFAULT_LOCKFILE_NAME; +use sysand_core::{ + commands::lock::DEFAULT_LOCKFILE_NAME, + env::local_directory::{DEFAULT_ENV_NAME, DEFAULT_MANIFEST_NAME, ENTRIES_PATH}, +}; // pub due to https://github.com/rust-lang/rust/issues/46379 mod common; pub use common::*; +#[test] +fn sync_to_current() -> Result<(), Box> { + let (_temp_dir, cwd, out) = run_sysand( + ["init", "--version", "1.2.3", "--name", "sync_to_current"], + None, + )?; + + out.assert().success(); + + { + let mut sysml_file = std::fs::File::create(cwd.join("test.sysml"))?; + sysml_file.write_all(b"package P;\n")?; + } + + let out = run_sysand_in(&cwd, ["include", "test.sysml"], None)?; + + out.assert().success(); + + let out = run_sysand_in(&cwd, ["sync"], None)?; + + out.assert() + .success() + .stderr(predicate::str::contains("Creating")) + .stderr(predicate::str::contains("Syncing")); + + let env_path = cwd.join(DEFAULT_ENV_NAME); + + let manifest = std::fs::read_to_string(env_path.join(DEFAULT_MANIFEST_NAME))?; + + assert_eq!( + manifest, + format!( + r#"[[project]] +name = "sync_to_current" +files = [ + "{}/test.sysml", +] +"#, + cwd + ) + ); + + let entries = std::fs::read_dir(env_path)?.collect::, _>>()?; + + assert_eq!(entries.len(), 2); + + assert_eq!(entries[0].file_name(), DEFAULT_MANIFEST_NAME); + + assert_eq!(entries[1].file_name(), ENTRIES_PATH); + + Ok(()) +} + #[test] fn sync_to_local() -> Result<(), Box> { let (_temp_dir, cwd) = new_temp_cwd()?; @@ -67,6 +123,19 @@ sources = [ .stderr(predicate::str::contains("Syncing")) .stderr(predicate::str::contains("Installing")); + let manifest = std::fs::read_to_string(cwd.join(DEFAULT_ENV_NAME).join(DEFAULT_MANIFEST_NAME))?; + + assert_eq!( + manifest, + format!( + r#"[[project]] +name = "sync_to_local" +directory = "{}/{DEFAULT_ENV_NAME}/5ddc0a2e8aaa88ac2bfc71aa0a8d08e020bceac4a90a4b72d8fb7f97ec5bfcc5/1.2.3.kpar" +"#, + cwd + ) + ); + let out = run_sysand_in(&cwd, ["env", "list"], None)?; out.assert() @@ -135,6 +204,19 @@ sources = [ info_mock.assert(); meta_mock.assert(); + let manifest = std::fs::read_to_string(cwd.join(DEFAULT_ENV_NAME).join(DEFAULT_MANIFEST_NAME))?; + + assert_eq!( + manifest, + format!( + r#"[[project]] +name = "sync_to_remote" +directory = "{}/{DEFAULT_ENV_NAME}/2b95cb7c6d6c08695b0e7c4b7e9d836c21de37fb9c72b0cfa26f53fd84a1b459/1.2.3.kpar" +"#, + cwd + ) + ); + let out = run_sysand_in(&cwd, ["env", "list"], None)?; out.assert() From 8991e3ecaa43a1de1aa71ebb9030b733708edee2 Mon Sep 17 00:00:00 2001 From: "victor.linroth.sensmetry" Date: Tue, 10 Feb 2026 21:04:48 +0100 Subject: [PATCH 2/4] Add optional publisher to manifest. Signed-off-by: victor.linroth.sensmetry --- Cargo.lock | 11 +++++++++++ core/Cargo.toml | 1 + core/src/env/local_directory.rs | 15 +++++++++++++-- core/src/lock.rs | 8 ++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2331472..f2a7254d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2462,6 +2462,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "packageurl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35da99768af1ae8830ccf30d295db0e09c24bcfda5a67515191dd4b773f6d82a" +dependencies = [ + "percent-encoding", + "thiserror 2.0.18", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -3428,6 +3438,7 @@ dependencies = [ "log", "logos", "mockito", + "packageurl", "port_check", "predicates", "pubgrub", diff --git a/core/Cargo.toml b/core/Cargo.toml index 371fa3cc..8c8cf2d8 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -61,6 +61,7 @@ tokio = { version = "1.48.0", default-features = false, features = ["rt", "io-ut bytes = { version = "1.11.0", default-features = false } toml_edit = { version = "0.23.9", features = ["serde"] } globset = { version = "0.4.18", default-features = false } +packageurl = "0.6.0" # Use native TLS only on Windows and Apple OSs [target.'cfg(any(target_os = "windows", target_vendor = "apple"))'.dependencies] diff --git a/core/src/env/local_directory.rs b/core/src/env/local_directory.rs index 2982bbac..5123634b 100644 --- a/core/src/env/local_directory.rs +++ b/core/src/env/local_directory.rs @@ -664,11 +664,17 @@ impl Lock { .filter_map(|usage| indices.get(&usage.resource)) .copied() .collect(); + let purl = project.get_package_url(); + let publisher = purl + .as_ref() + .and_then(|p| p.namespace().map(|ns| ns.to_owned())); + let name = purl.as_ref().map(|p| p.name().to_owned()).or(project.name); if let Some(storage) = storage { let directory = storage.root_path(); projects.push(ResolvedProject { - name: project.name, + publisher, + name, location: ResolvedLocation::Directory(directory), usages, }); @@ -681,7 +687,8 @@ impl Lock { .into_iter() .collect(); projects.push(ResolvedProject { - name: project.name, + publisher, + name, location: ResolvedLocation::Files(files), usages, }); @@ -724,6 +731,7 @@ pub enum ResolvedLocation { #[derive(Debug)] pub struct ResolvedProject { + pub publisher: Option, pub name: Option, pub location: ResolvedLocation, pub usages: Vec, @@ -732,6 +740,9 @@ pub struct ResolvedProject { impl ResolvedProject { pub fn to_toml(&self) -> Table { let mut table = Table::new(); + if let Some(publisher) = &self.publisher { + table.insert("publisher", value(publisher)); + } if let Some(name) = &self.name { table.insert("name", value(name)); } diff --git a/core/src/lock.rs b/core/src/lock.rs index e84b81cb..61b996ce 100644 --- a/core/src/lock.rs +++ b/core/src/lock.rs @@ -8,6 +8,7 @@ use std::{ str::FromStr, }; +use packageurl::PackageUrl; use semver::{Version, VersionReq}; use serde::Deserialize; use thiserror::Error; @@ -438,6 +439,13 @@ impl Project { table.insert("checksum", value(&self.checksum)); table } + + // Simple stopgap solution for now + pub fn get_package_url<'a>(&self) -> Option> { + self.identifiers + .first() + .and_then(|id| PackageUrl::from_str(id.as_str()).ok()) + } } const SOURCE_ENTRIES: &[&str] = &[ From c3b94fe126c62be430f672533225a9d6e7e9fcb0 Mon Sep 17 00:00:00 2001 From: "victor.linroth.sensmetry" Date: Wed, 11 Feb 2026 09:29:35 +0100 Subject: [PATCH 3/4] Split up `local_directory`. Signed-off-by: victor.linroth.sensmetry --- core/src/env/local_directory.rs | 770 ----------------------- core/src/env/local_directory/manifest.rs | 161 +++++ core/src/env/local_directory/mod.rs | 333 ++++++++++ core/src/env/local_directory/utils.rs | 309 +++++++++ 4 files changed, 803 insertions(+), 770 deletions(-) delete mode 100644 core/src/env/local_directory.rs create mode 100644 core/src/env/local_directory/manifest.rs create mode 100644 core/src/env/local_directory/mod.rs create mode 100644 core/src/env/local_directory/utils.rs diff --git a/core/src/env/local_directory.rs b/core/src/env/local_directory.rs deleted file mode 100644 index 5123634b..00000000 --- a/core/src/env/local_directory.rs +++ /dev/null @@ -1,770 +0,0 @@ -// SPDX-FileCopyrightText: © 2025 Sysand contributors -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::{ - collections::HashMap, - fmt::Display, - fs, - io::{self, BufRead, BufReader, Read, Write}, - num::TryFromIntError, -}; - -use camino::{Utf8Path, Utf8PathBuf}; -use camino_tempfile::NamedUtf8TempFile; -use sha2::Sha256; -use thiserror::Error; -use toml_edit::{Array, ArrayOfTables, DocumentMut, Item, Table, Value, value}; - -use crate::{ - commands::sources::{LocalSourcesError, do_sources_local_src_project_no_deps}, - env::{PutProjectError, ReadEnvironment, WriteEnvironment, segment_uri_generic}, - lock::{Lock, ResolutionError, Source, multiline_list}, - project::{ - local_src::{LocalSrcError, LocalSrcProject, PathError}, - utils::{ - FsIoError, ProjectDeserializationError, ProjectSerializationError, ToPathBuf, wrapfs, - }, - }, -}; - -#[derive(Clone, Debug)] -pub struct LocalDirectoryEnvironment { - pub environment_path: Utf8PathBuf, -} - -pub const DEFAULT_ENV_NAME: &str = "sysand_env"; - -pub const DEFAULT_MANIFEST_NAME: &str = "current.toml"; - -pub const ENTRIES_PATH: &str = "entries.txt"; -pub const VERSIONS_PATH: &str = "versions.txt"; - -/// Get a relative path corresponding to the given `uri` -pub fn path_encode_uri>(uri: S) -> Utf8PathBuf { - let mut result = Utf8PathBuf::new(); - for segment in segment_uri_generic::(uri) { - result.push(segment); - } - - result -} - -pub fn remove_dir_if_empty>(path: P) -> Result<(), FsIoError> { - match fs::remove_dir(path.as_ref()) { - Err(err) if err.kind() == io::ErrorKind::DirectoryNotEmpty => Ok(()), - r => r.map_err(|e| FsIoError::RmDir(path.to_path_buf(), e)), - } -} - -pub fn remove_empty_dirs>(path: P) -> Result<(), FsIoError> { - let mut dirs: Vec<_> = walkdir::WalkDir::new(path.as_ref()) - .into_iter() - .filter_map(|e| e.ok()) - .filter_map(|e| { - e.file_type() - .is_dir() - .then(|| Utf8PathBuf::from_path_buf(e.into_path()).ok()) - .flatten() - }) - .collect(); - - dirs.sort_by(|a, b| b.cmp(a)); - - for dir in dirs { - remove_dir_if_empty(&dir)?; - } - - Ok(()) -} - -#[derive(Error, Debug)] -pub enum TryMoveError { - #[error("recovered from failure: {0}")] - RecoveredIO(Box), - #[error( - "failed and may have left the directory in inconsistent state:\n{err}\nwhich was caused by:\n{cause}" - )] - CatastrophicIO { - err: Box, - cause: Box, - }, -} - -fn try_remove_files, I: Iterator>( - paths: I, -) -> Result<(), TryMoveError> { - let tempdir = camino_tempfile::tempdir() - .map_err(|e| TryMoveError::RecoveredIO(FsIoError::CreateTempFile(e).into()))?; - let mut moved: Vec = vec![]; - - for (i, path) in paths.enumerate() { - match move_fs_item(&path, tempdir.path().join(i.to_string())) { - Ok(_) => { - moved.push(path.to_path_buf()); - } - Err(cause) => { - // NOTE: This dance is to bypass the fact that std::io::error is not Clone-eable... - let mut catastrophic_error = None; - for (j, recover) in moved.iter().enumerate() { - if let Err(err) = move_fs_item(tempdir.path().join(j.to_string()), recover) { - catastrophic_error = Some(err); - break; - } - } - - if let Some(err) = catastrophic_error { - return Err(TryMoveError::CatastrophicIO { err, cause }); - } else { - return Err(TryMoveError::RecoveredIO(cause)); - } - } - } - } - - Ok(()) -} - -// Recursively copy a directory from `src` to `dst`. -// Assumes that all parents of `dst` exist. -fn copy_dir_recursive, Q: AsRef>( - src: P, - dst: Q, -) -> Result<(), Box> { - wrapfs::create_dir(&dst)?; - - for entry_result in wrapfs::read_dir(&src)? { - let entry = entry_result.map_err(|e| FsIoError::ReadDir(src.to_path_buf(), e))?; - let file_type = entry - .file_type() - .map_err(|e| FsIoError::ReadDir(src.to_path_buf(), e))?; - let src_path = entry.path(); - let dst_path = dst.as_ref().join(entry.file_name()); - - if file_type.is_dir() { - copy_dir_recursive(src_path, dst_path)?; - } else { - wrapfs::copy(src_path, dst_path)?; - } - } - - Ok(()) -} - -// Rename/move a file or directory from `src` to `dst`. -fn move_fs_item, Q: AsRef>( - src: P, - dst: Q, -) -> Result<(), Box> { - match fs::rename(src.as_ref(), dst.as_ref()) { - Ok(_) => Ok(()), - Err(e) if e.kind() == io::ErrorKind::CrossesDevices => { - let metadata = wrapfs::metadata(&src)?; - if metadata.is_dir() { - copy_dir_recursive(&src, &dst)?; - wrapfs::remove_dir_all(&src)?; - } else { - wrapfs::copy(&src, &dst)?; - wrapfs::remove_file(&src)?; - } - Ok(()) - } - Err(e) => Err(FsIoError::Move(src.to_path_buf(), dst.to_path_buf(), e))?, - } -} - -fn try_move_files(paths: &Vec<(&Utf8Path, &Utf8Path)>) -> Result<(), TryMoveError> { - let tempdir = camino_tempfile::tempdir() - .map_err(|e| TryMoveError::RecoveredIO(FsIoError::CreateTempFile(e).into()))?; - - let mut last_err = None; - - // move source files out of the way - for (i, (path, _)) in paths.iter().enumerate() { - let src_path = tempdir.path().join(format!("src_{}", i)); - if let Err(e) = move_fs_item(path, src_path) { - last_err = Some(e); - break; - } - } - - // Recover moved files in case of failure - if let Some(cause) = last_err { - for (i, (path, _)) in paths.iter().enumerate() { - let src_path = tempdir.path().join(format!("src_{}", i)); - - if src_path.exists() { - if let Err(err) = move_fs_item(src_path, path) { - return Err(TryMoveError::CatastrophicIO { err, cause }); - } - } - } - - return Err(TryMoveError::RecoveredIO(cause)); - } - - let mut last_err = None; - - // Move target files out of the way - for (i, (_, path)) in paths.iter().enumerate() { - if path.exists() { - let trg_path = tempdir.path().join(format!("trg_{}", i)); - if let Err(e) = move_fs_item(path, trg_path) { - last_err = Some(e); - break; - } - } - } - - // Recover moved files in case of failure - if let Some(cause) = last_err { - for (i, (_, path)) in paths.iter().enumerate() { - let trg_path = tempdir.path().join(format!("trg_{}", i)); - - if trg_path.exists() { - if let Err(err) = move_fs_item(trg_path, path) { - return Err(TryMoveError::CatastrophicIO { err, cause }); - } - } - } - - for (i, (path, _)) in paths.iter().enumerate() { - let src_path = tempdir.path().join(format!("src_{}", i)); - - if src_path.exists() { - if let Err(err) = move_fs_item(src_path, path) { - return Err(TryMoveError::CatastrophicIO { err, cause }); - } - } - } - - return Err(TryMoveError::RecoveredIO(cause)); - } - - let mut last_err = None; - - // Try moving files to destination - for (i, (_, target)) in paths.iter().enumerate() { - let src_path = tempdir.path().join(format!("src_{}", i)); - - if let Err(e) = move_fs_item(src_path, target) { - last_err = Some(e); - break; - } - } - - // Recover moved files in case of failure - if let Some(cause) = last_err { - for (i, (_, path)) in paths.iter().enumerate() { - let src_path = tempdir.path().join(format!("src_{}", i)); - - if path.exists() { - if let Err(err) = move_fs_item(path, src_path) { - return Err(TryMoveError::CatastrophicIO { err, cause }); - } - } - } - - for (i, (_, path)) in paths.iter().enumerate() { - let trg_path = tempdir.path().join(format!("trg_{}", i)); - - if trg_path.exists() { - if let Err(err) = move_fs_item(trg_path, path) { - return Err(TryMoveError::CatastrophicIO { err, cause }); - } - } - } - - for (i, (path, _)) in paths.iter().enumerate() { - let src_path = tempdir.path().join(format!("src_{}", i)); - - if src_path.exists() { - if let Err(err) = move_fs_item(src_path, path) { - return Err(TryMoveError::CatastrophicIO { err, cause }); - } - } - } - - return Err(TryMoveError::RecoveredIO(cause)); - } - - Ok(()) -} - -impl LocalDirectoryEnvironment { - pub fn root_path(&self) -> Utf8PathBuf { - self.environment_path.clone() - } - - pub fn entries_path(&self) -> Utf8PathBuf { - self.environment_path.join(ENTRIES_PATH) - } - - pub fn uri_path>(&self, uri: S) -> Utf8PathBuf { - self.environment_path.join(path_encode_uri(uri)) - } - - pub fn versions_path>(&self, uri: S) -> Utf8PathBuf { - let mut p = self.uri_path(uri); - p.push(VERSIONS_PATH); - p - } - - pub fn project_path, T: AsRef>(&self, uri: S, version: T) -> Utf8PathBuf { - let mut p = self.uri_path(uri); - p.push(format!("{}.kpar", version.as_ref())); - p - } -} - -#[derive(Error, Debug)] -pub enum LocalReadError { - #[error("failed to read project list file `entries.txt`: {0}")] - ProjectListFileRead(io::Error), - #[error("failed to read project versions file `versions.txt`: {0}")] - ProjectVersionsFileRead(io::Error), - #[error(transparent)] - Io(#[from] Box), -} - -impl From for LocalReadError { - fn from(v: FsIoError) -> Self { - Self::Io(Box::new(v)) - } -} - -impl ReadEnvironment for LocalDirectoryEnvironment { - type ReadError = LocalReadError; - - type UriIter = std::iter::Map< - io::Lines>, - fn(Result) -> Result, - >; - - fn uris(&self) -> Result { - Ok(BufReader::new(wrapfs::File::open(self.entries_path())?) - .lines() - .map(|x| match x { - Ok(line) => Ok(line), - Err(err) => Err(LocalReadError::ProjectListFileRead(err)), - })) - } - - type VersionIter = std::iter::Map< - io::Lines>, - fn(Result) -> Result, - >; - - fn versions>(&self, uri: S) -> Result { - let vp = self.versions_path(uri); - - // TODO: Better refactor the interface to return a - // maybe (similar to *Map::get) - if !vp.exists() { - if let Some(vpp) = vp.parent() { - if !vpp.exists() { - wrapfs::create_dir(vpp)?; - } - } - wrapfs::File::create(&vp)?; - } - - Ok(BufReader::new(wrapfs::File::open(&vp)?) - .lines() - .map(|x| match x { - Ok(line) => Ok(line), - Err(err) => Err(LocalReadError::ProjectVersionsFileRead(err)), - })) - } - - type InterchangeProjectRead = LocalSrcProject; - - fn get_project, T: AsRef>( - &self, - uri: S, - version: T, - ) -> Result { - let path = self.project_path(uri, version); - let project_path = wrapfs::canonicalize(path)?; - - Ok(LocalSrcProject { project_path }) - } -} - -#[derive(Error, Debug)] -pub enum LocalWriteError { - #[error(transparent)] - Deserialize(#[from] ProjectDeserializationError), - #[error(transparent)] - Serialize(#[from] ProjectSerializationError), - #[error("path error: {0}")] - Path(#[from] PathError), - #[error("already exists: {0}")] - AlreadyExists(String), - #[error(transparent)] - Io(#[from] Box), - #[error(transparent)] - TryMove(#[from] TryMoveError), - #[error(transparent)] - LocalRead(LocalReadError), -} - -impl From for LocalWriteError { - fn from(v: FsIoError) -> Self { - Self::Io(Box::new(v)) - } -} - -impl From for LocalWriteError { - fn from(value: LocalReadError) -> Self { - match value { - LocalReadError::Io(error) => Self::Io(error), - e @ (LocalReadError::ProjectListFileRead(_) - | LocalReadError::ProjectVersionsFileRead(_)) => Self::LocalRead(e), - } - } -} - -impl From for LocalWriteError { - fn from(value: LocalSrcError) -> Self { - match value { - LocalSrcError::Deserialize(error) => LocalWriteError::Deserialize(error), - LocalSrcError::Path(path_error) => LocalWriteError::Path(path_error), - LocalSrcError::AlreadyExists(msg) => LocalWriteError::AlreadyExists(msg), - LocalSrcError::Io(e) => LocalWriteError::Io(e), - LocalSrcError::Serialize(error) => Self::Serialize(error), - } - } -} - -fn add_line_temp>( - reader: R, - line: S, -) -> Result { - let mut temp_file = NamedUtf8TempFile::new().map_err(FsIoError::CreateTempFile)?; - - let mut line_added = false; - for this_line in BufReader::new(reader).lines() { - let this_line = this_line.map_err(|e| FsIoError::ReadFile(temp_file.to_path_buf(), e))?; - - if !line_added && line.as_ref() < this_line.as_str() { - writeln!(temp_file, "{}", line.as_ref()) - .map_err(|e| FsIoError::WriteFile(temp_file.path().into(), e))?; - line_added = true; - } - - writeln!(temp_file, "{}", this_line) - .map_err(|e| FsIoError::WriteFile(temp_file.path().into(), e))?; - - if line.as_ref() == this_line { - line_added = true; - } - } - - if !line_added { - writeln!(temp_file, "{}", line.as_ref()) - .map_err(|e| FsIoError::WriteFile(temp_file.path().into(), e))?; - } - - Ok(temp_file) -} - -fn singleton_line_temp>(line: S) -> Result { - let mut temp_file = NamedUtf8TempFile::new().map_err(FsIoError::CreateTempFile)?; - - writeln!(temp_file, "{}", line.as_ref()) - .map_err(|e| FsIoError::WriteFile(temp_file.path().into(), e))?; - - Ok(temp_file) -} - -impl WriteEnvironment for LocalDirectoryEnvironment { - type WriteError = LocalWriteError; - - type InterchangeProjectMut = LocalSrcProject; - - fn put_project, T: AsRef, F, E>( - &mut self, - uri: S, - version: T, - write_project: F, - ) -> Result> - where - F: FnOnce(&mut Self::InterchangeProjectMut) -> Result<(), E>, - { - let uri_path = self.uri_path(&uri); - let versions_path = self.versions_path(&uri); - - let entries_temp = add_line_temp( - wrapfs::File::open(self.entries_path()).map_err(LocalWriteError::from)?, - &uri, - )?; - - let versions_temp = if !versions_path.exists() { - singleton_line_temp(version.as_ref()) - } else { - let current_versions_f = - wrapfs::File::open(&versions_path).map_err(LocalWriteError::from)?; - add_line_temp(current_versions_f, version.as_ref()) - }?; - - let project_temp = camino_tempfile::tempdir() - .map_err(|e| LocalWriteError::from(FsIoError::MkTempDir(e)))?; - - let mut tentative_project = LocalSrcProject { - project_path: project_temp.path().to_path_buf(), - }; - - write_project(&mut tentative_project).map_err(PutProjectError::Callback)?; - - // Project write was successful - - if !uri_path.exists() { - wrapfs::create_dir(&uri_path).map_err(LocalWriteError::from)?; - } - - // Move existing stuff out of the way - let project_path = self.project_path(&uri, &version); - - // TODO: Handle catastrophic errors differently - try_move_files(&vec![ - (project_temp.path(), &project_path), - (versions_temp.path(), &versions_path), - (entries_temp.path(), &self.entries_path()), - ]) - .map_err(LocalWriteError::from)?; - - Ok(LocalSrcProject { project_path }) - } - - fn del_project_version, T: AsRef>( - &mut self, - uri: S, - version: T, - ) -> Result<(), Self::WriteError> { - let mut versions_temp = - NamedUtf8TempFile::with_suffix("versions.txt").map_err(FsIoError::CreateTempFile)?; - - let versions_path = self.versions_path(&uri); - let mut found = false; - let mut empty = true; - - // I think this may be needed on Windows in order to drop the - // file handle before overwriting - { - let current_versions_f = BufReader::new(wrapfs::File::open(&versions_path)?); - for version_line_ in current_versions_f.lines() { - let version_line = version_line_ - .map_err(|e| FsIoError::ReadFile(versions_path.to_path_buf(), e))?; - - if version.as_ref() != version_line { - writeln!(versions_temp, "{}", version_line) - .map_err(|e| FsIoError::WriteFile(versions_path.clone(), e))?; - - empty = false; - } else { - found = true; - } - } - } - - if found { - let project: LocalSrcProject = self - .get_project(&uri, version) - .map_err(LocalWriteError::from)?; - - // TODO: Add better error messages for catastrophic errors - if let Err(err) = try_remove_files(project.get_source_paths()?.into_iter().chain(vec![ - project.project_path.join(".project.json"), - project.project_path.join(".meta.json"), - ])) { - match err { - TryMoveError::CatastrophicIO { .. } => { - // Censor the version if a partial delete happened, better pretend - // like it does not exist than to pretend like a broken - // package is properly installed - wrapfs::copy(versions_temp.path(), &versions_path)?; - return Err(err.into()); - } - TryMoveError::RecoveredIO(_) => return Err(LocalWriteError::from(err)), - } - } - - wrapfs::copy(versions_temp.path(), &versions_path)?; - - remove_empty_dirs(project.project_path)?; - if empty { - let current_uris_: Result, LocalReadError> = self.uris()?.collect(); - let current_uris: Vec = current_uris_?; - let entries_path = self.entries_path(); - let mut f = io::BufWriter::new(wrapfs::File::create(&entries_path)?); - for existing_uri in current_uris { - if uri.as_ref() != existing_uri { - writeln!(f, "{}", existing_uri) - .map_err(|e| FsIoError::WriteFile(entries_path.clone(), e))?; - } - } - wrapfs::remove_file(versions_path)?; - remove_dir_if_empty(self.uri_path(&uri))?; - } - } - - Ok(()) - } - - fn del_uri>(&mut self, uri: S) -> Result<(), Self::WriteError> { - let current_uris_: Result, LocalReadError> = self.uris()?.collect(); - let current_uris: Vec = current_uris_?; - - if current_uris.contains(&uri.as_ref().to_string()) { - for version_ in self.versions(&uri)? { - let version: String = version_?; - self.del_project_version(&uri, &version)?; - } - } - - Ok(()) - } -} - -#[derive(Debug, Error)] -pub enum ResolvedManifestError { - #[error(transparent)] - ResolutionError(#[from] ResolutionError), - #[error("too many dependencies, unable to convert to i64: {0}")] - TooManyDependencies(TryFromIntError), - #[error(transparent)] - LocalSources(#[from] LocalSourcesError), - #[error(transparent)] - Canonicalization(#[from] Box), -} - -impl Lock { - pub fn to_resolved_manifest>( - &self, - env: &LocalDirectoryEnvironment, - root_path: P, - ) -> Result { - let resolved_projects = self.resolve_projects(env)?; - - let indices = resolved_projects - .iter() - .map(|(p, _)| p) - .enumerate() - .flat_map(|(num, p)| p.identifiers.iter().map(move |iri| (iri.clone(), num))) - .map(|(iri, num)| i64::try_from(num).map(|num| (iri, num))) - .collect::, _>>() - .map_err(ResolvedManifestError::TooManyDependencies)?; - let indices = HashMap::::from_iter(indices); - - let mut projects = vec![]; - for (project, storage) in resolved_projects { - let usages = project - .usages - .iter() - .filter_map(|usage| indices.get(&usage.resource)) - .copied() - .collect(); - let purl = project.get_package_url(); - let publisher = purl - .as_ref() - .and_then(|p| p.namespace().map(|ns| ns.to_owned())); - let name = purl.as_ref().map(|p| p.name().to_owned()).or(project.name); - - if let Some(storage) = storage { - let directory = storage.root_path(); - projects.push(ResolvedProject { - publisher, - name, - location: ResolvedLocation::Directory(directory), - usages, - }); - } else if let [Source::Editable { editable }, ..] = project.sources.as_slice() { - let project_path = root_path.as_ref().join(editable.as_str()); - let editable_project = LocalSrcProject { - project_path: wrapfs::canonicalize(project_path)?, - }; - let files = do_sources_local_src_project_no_deps(&editable_project, true)? - .into_iter() - .collect(); - projects.push(ResolvedProject { - publisher, - name, - location: ResolvedLocation::Files(files), - usages, - }); - } - } - - Ok(ResolvedManifest { projects }) - } -} - -#[derive(Debug)] -pub struct ResolvedManifest { - pub projects: Vec, -} - -impl Display for ResolvedManifest { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.to_toml()) - } -} - -impl ResolvedManifest { - pub fn to_toml(&self) -> DocumentMut { - let mut doc = DocumentMut::new(); - let mut projects = ArrayOfTables::new(); - for project in &self.projects { - projects.push(project.to_toml()); - } - doc.insert("project", Item::ArrayOfTables(projects)); - - doc - } -} - -#[derive(Debug)] -pub enum ResolvedLocation { - Directory(Utf8PathBuf), - Files(Vec), -} - -#[derive(Debug)] -pub struct ResolvedProject { - pub publisher: Option, - pub name: Option, - pub location: ResolvedLocation, - pub usages: Vec, -} - -impl ResolvedProject { - pub fn to_toml(&self) -> Table { - let mut table = Table::new(); - if let Some(publisher) = &self.publisher { - table.insert("publisher", value(publisher)); - } - if let Some(name) = &self.name { - table.insert("name", value(name)); - } - match &self.location { - ResolvedLocation::Directory(dir) => { - table.insert("directory", value(dir.as_str())); - } - ResolvedLocation::Files(files) => { - if !files.is_empty() { - table.insert( - "files", - value(multiline_list( - files.iter().map(|f| Value::from(f.as_str())), - )), - ); - } - } - } - if !self.usages.is_empty() { - let usages = Array::from_iter(self.usages.iter().copied().map(Value::from)); - table.insert("usages", value(usages)); - } - table - } -} diff --git a/core/src/env/local_directory/manifest.rs b/core/src/env/local_directory/manifest.rs new file mode 100644 index 00000000..b61d0972 --- /dev/null +++ b/core/src/env/local_directory/manifest.rs @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: © 2025 Sysand contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{collections::HashMap, fmt::Display, num::TryFromIntError}; + +use camino::{Utf8Path, Utf8PathBuf}; +use thiserror::Error; +use toml_edit::{Array, ArrayOfTables, DocumentMut, Item, Table, Value, value}; + +use crate::{ + commands::sources::{LocalSourcesError, do_sources_local_src_project_no_deps}, + env::local_directory::{LocalDirectoryEnvironment, LocalReadError}, + lock::{Lock, ResolutionError, Source, multiline_list}, + project::{ + local_src::LocalSrcProject, + utils::{FsIoError, wrapfs}, + }, +}; + +#[derive(Debug, Error)] +pub enum ResolvedManifestError { + #[error(transparent)] + ResolutionError(#[from] ResolutionError), + #[error("too many dependencies, unable to convert to i64: {0}")] + TooManyDependencies(TryFromIntError), + #[error(transparent)] + LocalSources(#[from] LocalSourcesError), + #[error(transparent)] + Canonicalization(#[from] Box), +} + +impl Lock { + pub fn to_resolved_manifest>( + &self, + env: &LocalDirectoryEnvironment, + root_path: P, + ) -> Result { + let resolved_projects = self.resolve_projects(env)?; + + let indices = resolved_projects + .iter() + .map(|(p, _)| p) + .enumerate() + .flat_map(|(num, p)| p.identifiers.iter().map(move |iri| (iri.clone(), num))) + .map(|(iri, num)| i64::try_from(num).map(|num| (iri, num))) + .collect::, _>>() + .map_err(ResolvedManifestError::TooManyDependencies)?; + let indices = HashMap::::from_iter(indices); + + let mut projects = vec![]; + for (project, storage) in resolved_projects { + let usages = project + .usages + .iter() + .filter_map(|usage| indices.get(&usage.resource)) + .copied() + .collect(); + let purl = project.get_package_url(); + let publisher = purl + .as_ref() + .and_then(|p| p.namespace().map(|ns| ns.to_owned())); + let name = purl.as_ref().map(|p| p.name().to_owned()).or(project.name); + + if let Some(storage) = storage { + let directory = storage.root_path(); + projects.push(ResolvedProject { + publisher, + name, + location: ResolvedLocation::Directory(directory), + usages, + }); + } else if let [Source::Editable { editable }, ..] = project.sources.as_slice() { + let project_path = root_path.as_ref().join(editable.as_str()); + let editable_project = LocalSrcProject { + project_path: wrapfs::canonicalize(project_path)?, + }; + let files = do_sources_local_src_project_no_deps(&editable_project, true)? + .into_iter() + .collect(); + projects.push(ResolvedProject { + publisher, + name, + location: ResolvedLocation::Files(files), + usages, + }); + } + } + + Ok(ResolvedManifest { projects }) + } +} + +#[derive(Debug)] +pub struct ResolvedManifest { + pub projects: Vec, +} + +impl Display for ResolvedManifest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.to_toml()) + } +} + +impl ResolvedManifest { + pub fn to_toml(&self) -> DocumentMut { + let mut doc = DocumentMut::new(); + let mut projects = ArrayOfTables::new(); + for project in &self.projects { + projects.push(project.to_toml()); + } + doc.insert("project", Item::ArrayOfTables(projects)); + + doc + } +} + +#[derive(Debug)] +pub enum ResolvedLocation { + Directory(Utf8PathBuf), + Files(Vec), +} + +#[derive(Debug)] +pub struct ResolvedProject { + pub publisher: Option, + pub name: Option, + pub location: ResolvedLocation, + pub usages: Vec, +} + +impl ResolvedProject { + pub fn to_toml(&self) -> Table { + let mut table = Table::new(); + if let Some(publisher) = &self.publisher { + table.insert("publisher", value(publisher)); + } + if let Some(name) = &self.name { + table.insert("name", value(name)); + } + match &self.location { + ResolvedLocation::Directory(dir) => { + table.insert("directory", value(dir.as_str())); + } + ResolvedLocation::Files(files) => { + if !files.is_empty() { + table.insert( + "files", + value(multiline_list( + files.iter().map(|f| Value::from(f.as_str())), + )), + ); + } + } + } + if !self.usages.is_empty() { + let usages = Array::from_iter(self.usages.iter().copied().map(Value::from)); + table.insert("usages", value(usages)); + } + table + } +} diff --git a/core/src/env/local_directory/mod.rs b/core/src/env/local_directory/mod.rs new file mode 100644 index 00000000..fdc298b2 --- /dev/null +++ b/core/src/env/local_directory/mod.rs @@ -0,0 +1,333 @@ +// SPDX-FileCopyrightText: © 2025 Sysand contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::io::{self, BufRead, BufReader, Write}; + +use camino::Utf8PathBuf; +use camino_tempfile::NamedUtf8TempFile; +use thiserror::Error; + +use crate::{ + env::{PutProjectError, ReadEnvironment, WriteEnvironment}, + project::{ + local_src::{LocalSrcError, LocalSrcProject, PathError}, + utils::{ + FsIoError, ProjectDeserializationError, ProjectSerializationError, ToPathBuf, wrapfs, + }, + }, +}; + +pub mod manifest; +mod utils; + +use utils::{ + TryMoveError, add_line_temp, path_encode_uri, remove_dir_if_empty, remove_empty_dirs, + singleton_line_temp, try_move_files, try_remove_files, +}; + +#[derive(Clone, Debug)] +pub struct LocalDirectoryEnvironment { + pub environment_path: Utf8PathBuf, +} + +pub const DEFAULT_ENV_NAME: &str = "sysand_env"; + +pub const DEFAULT_MANIFEST_NAME: &str = "current.toml"; + +pub const ENTRIES_PATH: &str = "entries.txt"; +pub const VERSIONS_PATH: &str = "versions.txt"; + +impl LocalDirectoryEnvironment { + pub fn root_path(&self) -> Utf8PathBuf { + self.environment_path.clone() + } + + pub fn entries_path(&self) -> Utf8PathBuf { + self.environment_path.join(ENTRIES_PATH) + } + + pub fn uri_path>(&self, uri: S) -> Utf8PathBuf { + self.environment_path.join(path_encode_uri(uri)) + } + + pub fn versions_path>(&self, uri: S) -> Utf8PathBuf { + let mut p = self.uri_path(uri); + p.push(VERSIONS_PATH); + p + } + + pub fn project_path, T: AsRef>(&self, uri: S, version: T) -> Utf8PathBuf { + let mut p = self.uri_path(uri); + p.push(format!("{}.kpar", version.as_ref())); + p + } +} + +#[derive(Error, Debug)] +pub enum LocalReadError { + #[error("failed to read project list file `entries.txt`: {0}")] + ProjectListFileRead(io::Error), + #[error("failed to read project versions file `versions.txt`: {0}")] + ProjectVersionsFileRead(io::Error), + #[error(transparent)] + Io(#[from] Box), +} + +impl From for LocalReadError { + fn from(v: FsIoError) -> Self { + Self::Io(Box::new(v)) + } +} + +impl ReadEnvironment for LocalDirectoryEnvironment { + type ReadError = LocalReadError; + + type UriIter = std::iter::Map< + io::Lines>, + fn(Result) -> Result, + >; + + fn uris(&self) -> Result { + Ok(BufReader::new(wrapfs::File::open(self.entries_path())?) + .lines() + .map(|x| match x { + Ok(line) => Ok(line), + Err(err) => Err(LocalReadError::ProjectListFileRead(err)), + })) + } + + type VersionIter = std::iter::Map< + io::Lines>, + fn(Result) -> Result, + >; + + fn versions>(&self, uri: S) -> Result { + let vp = self.versions_path(uri); + + // TODO: Better refactor the interface to return a + // maybe (similar to *Map::get) + if !vp.exists() { + if let Some(vpp) = vp.parent() { + if !vpp.exists() { + wrapfs::create_dir(vpp)?; + } + } + wrapfs::File::create(&vp)?; + } + + Ok(BufReader::new(wrapfs::File::open(&vp)?) + .lines() + .map(|x| match x { + Ok(line) => Ok(line), + Err(err) => Err(LocalReadError::ProjectVersionsFileRead(err)), + })) + } + + type InterchangeProjectRead = LocalSrcProject; + + fn get_project, T: AsRef>( + &self, + uri: S, + version: T, + ) -> Result { + let path = self.project_path(uri, version); + let project_path = wrapfs::canonicalize(path)?; + + Ok(LocalSrcProject { project_path }) + } +} + +#[derive(Error, Debug)] +pub enum LocalWriteError { + #[error(transparent)] + Deserialize(#[from] ProjectDeserializationError), + #[error(transparent)] + Serialize(#[from] ProjectSerializationError), + #[error("path error: {0}")] + Path(#[from] PathError), + #[error("already exists: {0}")] + AlreadyExists(String), + #[error(transparent)] + Io(#[from] Box), + #[error(transparent)] + TryMove(#[from] TryMoveError), + #[error(transparent)] + LocalRead(LocalReadError), +} + +impl From for LocalWriteError { + fn from(v: FsIoError) -> Self { + Self::Io(Box::new(v)) + } +} + +impl From for LocalWriteError { + fn from(value: LocalReadError) -> Self { + match value { + LocalReadError::Io(error) => Self::Io(error), + e @ (LocalReadError::ProjectListFileRead(_) + | LocalReadError::ProjectVersionsFileRead(_)) => Self::LocalRead(e), + } + } +} + +impl From for LocalWriteError { + fn from(value: LocalSrcError) -> Self { + match value { + LocalSrcError::Deserialize(error) => LocalWriteError::Deserialize(error), + LocalSrcError::Path(path_error) => LocalWriteError::Path(path_error), + LocalSrcError::AlreadyExists(msg) => LocalWriteError::AlreadyExists(msg), + LocalSrcError::Io(e) => LocalWriteError::Io(e), + LocalSrcError::Serialize(error) => Self::Serialize(error), + } + } +} + +impl WriteEnvironment for LocalDirectoryEnvironment { + type WriteError = LocalWriteError; + + type InterchangeProjectMut = LocalSrcProject; + + fn put_project, T: AsRef, F, E>( + &mut self, + uri: S, + version: T, + write_project: F, + ) -> Result> + where + F: FnOnce(&mut Self::InterchangeProjectMut) -> Result<(), E>, + { + let uri_path = self.uri_path(&uri); + let versions_path = self.versions_path(&uri); + + let entries_temp = add_line_temp( + wrapfs::File::open(self.entries_path()).map_err(LocalWriteError::from)?, + &uri, + )?; + + let versions_temp = if !versions_path.exists() { + singleton_line_temp(version.as_ref()) + } else { + let current_versions_f = + wrapfs::File::open(&versions_path).map_err(LocalWriteError::from)?; + add_line_temp(current_versions_f, version.as_ref()) + }?; + + let project_temp = camino_tempfile::tempdir() + .map_err(|e| LocalWriteError::from(FsIoError::MkTempDir(e)))?; + + let mut tentative_project = LocalSrcProject { + project_path: project_temp.path().to_path_buf(), + }; + + write_project(&mut tentative_project).map_err(PutProjectError::Callback)?; + + // Project write was successful + + if !uri_path.exists() { + wrapfs::create_dir(&uri_path).map_err(LocalWriteError::from)?; + } + + // Move existing stuff out of the way + let project_path = self.project_path(&uri, &version); + + // TODO: Handle catastrophic errors differently + try_move_files(&vec![ + (project_temp.path(), &project_path), + (versions_temp.path(), &versions_path), + (entries_temp.path(), &self.entries_path()), + ]) + .map_err(LocalWriteError::from)?; + + Ok(LocalSrcProject { project_path }) + } + + fn del_project_version, T: AsRef>( + &mut self, + uri: S, + version: T, + ) -> Result<(), Self::WriteError> { + let mut versions_temp = + NamedUtf8TempFile::with_suffix("versions.txt").map_err(FsIoError::CreateTempFile)?; + + let versions_path = self.versions_path(&uri); + let mut found = false; + let mut empty = true; + + // I think this may be needed on Windows in order to drop the + // file handle before overwriting + { + let current_versions_f = BufReader::new(wrapfs::File::open(&versions_path)?); + for version_line_ in current_versions_f.lines() { + let version_line = version_line_ + .map_err(|e| FsIoError::ReadFile(versions_path.to_path_buf(), e))?; + + if version.as_ref() != version_line { + writeln!(versions_temp, "{}", version_line) + .map_err(|e| FsIoError::WriteFile(versions_path.clone(), e))?; + + empty = false; + } else { + found = true; + } + } + } + + if found { + let project: LocalSrcProject = self + .get_project(&uri, version) + .map_err(LocalWriteError::from)?; + + // TODO: Add better error messages for catastrophic errors + if let Err(err) = try_remove_files(project.get_source_paths()?.into_iter().chain(vec![ + project.project_path.join(".project.json"), + project.project_path.join(".meta.json"), + ])) { + match err { + TryMoveError::CatastrophicIO { .. } => { + // Censor the version if a partial delete happened, better pretend + // like it does not exist than to pretend like a broken + // package is properly installed + wrapfs::copy(versions_temp.path(), &versions_path)?; + return Err(err.into()); + } + TryMoveError::RecoveredIO(_) => return Err(LocalWriteError::from(err)), + } + } + + wrapfs::copy(versions_temp.path(), &versions_path)?; + + remove_empty_dirs(project.project_path)?; + if empty { + let current_uris_: Result, LocalReadError> = self.uris()?.collect(); + let current_uris: Vec = current_uris_?; + let entries_path = self.entries_path(); + let mut f = io::BufWriter::new(wrapfs::File::create(&entries_path)?); + for existing_uri in current_uris { + if uri.as_ref() != existing_uri { + writeln!(f, "{}", existing_uri) + .map_err(|e| FsIoError::WriteFile(entries_path.clone(), e))?; + } + } + wrapfs::remove_file(versions_path)?; + remove_dir_if_empty(self.uri_path(&uri))?; + } + } + + Ok(()) + } + + fn del_uri>(&mut self, uri: S) -> Result<(), Self::WriteError> { + let current_uris_: Result, LocalReadError> = self.uris()?.collect(); + let current_uris: Vec = current_uris_?; + + if current_uris.contains(&uri.as_ref().to_string()) { + for version_ in self.versions(&uri)? { + let version: String = version_?; + self.del_project_version(&uri, &version)?; + } + } + + Ok(()) + } +} diff --git a/core/src/env/local_directory/utils.rs b/core/src/env/local_directory/utils.rs new file mode 100644 index 00000000..3ac7509c --- /dev/null +++ b/core/src/env/local_directory/utils.rs @@ -0,0 +1,309 @@ +// SPDX-FileCopyrightText: © 2025 Sysand contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{ + fs, + io::{self, BufRead, BufReader, Read, Write}, +}; + +use camino::{Utf8Path, Utf8PathBuf}; +use camino_tempfile::NamedUtf8TempFile; +use sha2::Sha256; +use thiserror::Error; + +use crate::{ + env::{local_directory::LocalWriteError, segment_uri_generic}, + project::utils::{FsIoError, ToPathBuf, wrapfs}, +}; + +/// Get a relative path corresponding to the given `uri` +pub fn path_encode_uri>(uri: S) -> Utf8PathBuf { + let mut result = Utf8PathBuf::new(); + for segment in segment_uri_generic::(uri) { + result.push(segment); + } + + result +} + +pub fn remove_dir_if_empty>(path: P) -> Result<(), FsIoError> { + match fs::remove_dir(path.as_ref()) { + Err(err) if err.kind() == io::ErrorKind::DirectoryNotEmpty => Ok(()), + r => r.map_err(|e| FsIoError::RmDir(path.to_path_buf(), e)), + } +} + +pub fn remove_empty_dirs>(path: P) -> Result<(), FsIoError> { + let mut dirs: Vec<_> = walkdir::WalkDir::new(path.as_ref()) + .into_iter() + .filter_map(|e| e.ok()) + .filter_map(|e| { + e.file_type() + .is_dir() + .then(|| Utf8PathBuf::from_path_buf(e.into_path()).ok()) + .flatten() + }) + .collect(); + + dirs.sort_by(|a, b| b.cmp(a)); + + for dir in dirs { + remove_dir_if_empty(&dir)?; + } + + Ok(()) +} + +#[derive(Error, Debug)] +pub enum TryMoveError { + #[error("recovered from failure: {0}")] + RecoveredIO(Box), + #[error( + "failed and may have left the directory in inconsistent state:\n{err}\nwhich was caused by:\n{cause}" + )] + CatastrophicIO { + err: Box, + cause: Box, + }, +} + +pub fn try_remove_files, I: Iterator>( + paths: I, +) -> Result<(), TryMoveError> { + let tempdir = camino_tempfile::tempdir() + .map_err(|e| TryMoveError::RecoveredIO(FsIoError::CreateTempFile(e).into()))?; + let mut moved: Vec = vec![]; + + for (i, path) in paths.enumerate() { + match move_fs_item(&path, tempdir.path().join(i.to_string())) { + Ok(_) => { + moved.push(path.to_path_buf()); + } + Err(cause) => { + // NOTE: This dance is to bypass the fact that std::io::error is not Clone-eable... + let mut catastrophic_error = None; + for (j, recover) in moved.iter().enumerate() { + if let Err(err) = move_fs_item(tempdir.path().join(j.to_string()), recover) { + catastrophic_error = Some(err); + break; + } + } + + if let Some(err) = catastrophic_error { + return Err(TryMoveError::CatastrophicIO { err, cause }); + } else { + return Err(TryMoveError::RecoveredIO(cause)); + } + } + } + } + + Ok(()) +} + +// Recursively copy a directory from `src` to `dst`. +// Assumes that all parents of `dst` exist. +fn copy_dir_recursive, Q: AsRef>( + src: P, + dst: Q, +) -> Result<(), Box> { + wrapfs::create_dir(&dst)?; + + for entry_result in wrapfs::read_dir(&src)? { + let entry = entry_result.map_err(|e| FsIoError::ReadDir(src.to_path_buf(), e))?; + let file_type = entry + .file_type() + .map_err(|e| FsIoError::ReadDir(src.to_path_buf(), e))?; + let src_path = entry.path(); + let dst_path = dst.as_ref().join(entry.file_name()); + + if file_type.is_dir() { + copy_dir_recursive(src_path, dst_path)?; + } else { + wrapfs::copy(src_path, dst_path)?; + } + } + + Ok(()) +} + +// Rename/move a file or directory from `src` to `dst`. +fn move_fs_item, Q: AsRef>( + src: P, + dst: Q, +) -> Result<(), Box> { + match fs::rename(src.as_ref(), dst.as_ref()) { + Ok(_) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::CrossesDevices => { + let metadata = wrapfs::metadata(&src)?; + if metadata.is_dir() { + copy_dir_recursive(&src, &dst)?; + wrapfs::remove_dir_all(&src)?; + } else { + wrapfs::copy(&src, &dst)?; + wrapfs::remove_file(&src)?; + } + Ok(()) + } + Err(e) => Err(FsIoError::Move(src.to_path_buf(), dst.to_path_buf(), e))?, + } +} + +pub fn try_move_files(paths: &Vec<(&Utf8Path, &Utf8Path)>) -> Result<(), TryMoveError> { + let tempdir = camino_tempfile::tempdir() + .map_err(|e| TryMoveError::RecoveredIO(FsIoError::CreateTempFile(e).into()))?; + + let mut last_err = None; + + // move source files out of the way + for (i, (path, _)) in paths.iter().enumerate() { + let src_path = tempdir.path().join(format!("src_{}", i)); + if let Err(e) = move_fs_item(path, src_path) { + last_err = Some(e); + break; + } + } + + // Recover moved files in case of failure + if let Some(cause) = last_err { + for (i, (path, _)) in paths.iter().enumerate() { + let src_path = tempdir.path().join(format!("src_{}", i)); + + if src_path.exists() { + if let Err(err) = move_fs_item(src_path, path) { + return Err(TryMoveError::CatastrophicIO { err, cause }); + } + } + } + + return Err(TryMoveError::RecoveredIO(cause)); + } + + let mut last_err = None; + + // Move target files out of the way + for (i, (_, path)) in paths.iter().enumerate() { + if path.exists() { + let trg_path = tempdir.path().join(format!("trg_{}", i)); + if let Err(e) = move_fs_item(path, trg_path) { + last_err = Some(e); + break; + } + } + } + + // Recover moved files in case of failure + if let Some(cause) = last_err { + for (i, (_, path)) in paths.iter().enumerate() { + let trg_path = tempdir.path().join(format!("trg_{}", i)); + + if trg_path.exists() { + if let Err(err) = move_fs_item(trg_path, path) { + return Err(TryMoveError::CatastrophicIO { err, cause }); + } + } + } + + for (i, (path, _)) in paths.iter().enumerate() { + let src_path = tempdir.path().join(format!("src_{}", i)); + + if src_path.exists() { + if let Err(err) = move_fs_item(src_path, path) { + return Err(TryMoveError::CatastrophicIO { err, cause }); + } + } + } + + return Err(TryMoveError::RecoveredIO(cause)); + } + + let mut last_err = None; + + // Try moving files to destination + for (i, (_, target)) in paths.iter().enumerate() { + let src_path = tempdir.path().join(format!("src_{}", i)); + + if let Err(e) = move_fs_item(src_path, target) { + last_err = Some(e); + break; + } + } + + // Recover moved files in case of failure + if let Some(cause) = last_err { + for (i, (_, path)) in paths.iter().enumerate() { + let src_path = tempdir.path().join(format!("src_{}", i)); + + if path.exists() { + if let Err(err) = move_fs_item(path, src_path) { + return Err(TryMoveError::CatastrophicIO { err, cause }); + } + } + } + + for (i, (_, path)) in paths.iter().enumerate() { + let trg_path = tempdir.path().join(format!("trg_{}", i)); + + if trg_path.exists() { + if let Err(err) = move_fs_item(trg_path, path) { + return Err(TryMoveError::CatastrophicIO { err, cause }); + } + } + } + + for (i, (path, _)) in paths.iter().enumerate() { + let src_path = tempdir.path().join(format!("src_{}", i)); + + if src_path.exists() { + if let Err(err) = move_fs_item(src_path, path) { + return Err(TryMoveError::CatastrophicIO { err, cause }); + } + } + } + + return Err(TryMoveError::RecoveredIO(cause)); + } + + Ok(()) +} + +pub fn add_line_temp>( + reader: R, + line: S, +) -> Result { + let mut temp_file = NamedUtf8TempFile::new().map_err(FsIoError::CreateTempFile)?; + + let mut line_added = false; + for this_line in BufReader::new(reader).lines() { + let this_line = this_line.map_err(|e| FsIoError::ReadFile(temp_file.to_path_buf(), e))?; + + if !line_added && line.as_ref() < this_line.as_str() { + writeln!(temp_file, "{}", line.as_ref()) + .map_err(|e| FsIoError::WriteFile(temp_file.path().into(), e))?; + line_added = true; + } + + writeln!(temp_file, "{}", this_line) + .map_err(|e| FsIoError::WriteFile(temp_file.path().into(), e))?; + + if line.as_ref() == this_line { + line_added = true; + } + } + + if !line_added { + writeln!(temp_file, "{}", line.as_ref()) + .map_err(|e| FsIoError::WriteFile(temp_file.path().into(), e))?; + } + + Ok(temp_file) +} + +pub fn singleton_line_temp>(line: S) -> Result { + let mut temp_file = NamedUtf8TempFile::new().map_err(FsIoError::CreateTempFile)?; + + writeln!(temp_file, "{}", line.as_ref()) + .map_err(|e| FsIoError::WriteFile(temp_file.path().into(), e))?; + + Ok(temp_file) +} From 2ff1dba6068f5b77b6250c244757aaea54763ab8 Mon Sep 17 00:00:00 2001 From: "victor.linroth.sensmetry" Date: Wed, 11 Feb 2026 12:51:52 +0100 Subject: [PATCH 4/4] Deserialize empty arrays. Signed-off-by: victor.linroth.sensmetry --- core/src/env/local_directory/manifest.rs | 16 ++++++---------- sysand/tests/cli_sync.rs | 3 +++ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/core/src/env/local_directory/manifest.rs b/core/src/env/local_directory/manifest.rs index b61d0972..baae40d4 100644 --- a/core/src/env/local_directory/manifest.rs +++ b/core/src/env/local_directory/manifest.rs @@ -142,20 +142,16 @@ impl ResolvedProject { table.insert("directory", value(dir.as_str())); } ResolvedLocation::Files(files) => { + let file_iter = files.iter().map(|f| Value::from(f.as_str())); if !files.is_empty() { - table.insert( - "files", - value(multiline_list( - files.iter().map(|f| Value::from(f.as_str())), - )), - ); + table.insert("files", value(multiline_list(file_iter))); + } else { + table.insert("files", value(Array::from_iter(file_iter))); } } } - if !self.usages.is_empty() { - let usages = Array::from_iter(self.usages.iter().copied().map(Value::from)); - table.insert("usages", value(usages)); - } + let usages = Array::from_iter(self.usages.iter().copied().map(Value::from)); + table.insert("usages", value(usages)); table } } diff --git a/sysand/tests/cli_sync.rs b/sysand/tests/cli_sync.rs index fc9c4eae..4cb9445c 100644 --- a/sysand/tests/cli_sync.rs +++ b/sysand/tests/cli_sync.rs @@ -53,6 +53,7 @@ name = "sync_to_current" files = [ "{}/test.sysml", ] +usages = [] "#, cwd ) @@ -131,6 +132,7 @@ sources = [ r#"[[project]] name = "sync_to_local" directory = "{}/{DEFAULT_ENV_NAME}/5ddc0a2e8aaa88ac2bfc71aa0a8d08e020bceac4a90a4b72d8fb7f97ec5bfcc5/1.2.3.kpar" +usages = [] "#, cwd ) @@ -212,6 +214,7 @@ sources = [ r#"[[project]] name = "sync_to_remote" directory = "{}/{DEFAULT_ENV_NAME}/2b95cb7c6d6c08695b0e7c4b7e9d836c21de37fb9c72b0cfa26f53fd84a1b459/1.2.3.kpar" +usages = [] "#, cwd )