diff --git a/Cargo.lock b/Cargo.lock index 963710b..ab2c168 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,9 +228,15 @@ name = "componentize-go" version = "0.2.0" dependencies = [ "anyhow", + "bzip2", "clap", + "dirs", "regex", - "wat", + "reqwest", + "serde", + "tar", + "toml", + "which", "wit-bindgen-go", "wit-component", "wit-parser", @@ -262,6 +268,27 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -847,6 +874,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1001,6 +1034,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror 2.0.17", +] + [[package]] name = "regex" version = "1.12.3" @@ -1274,6 +1318,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98" +dependencies = [ + "serde_core", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1510,6 +1563,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8195ca05e4eb728f4ba94f3e3291661320af739c4e43779cbdfae82ab239fcc" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d282ade6016312faf3e41e57ebbba0c073e4056dab1232ab1cb624199648f8ed" + [[package]] name = "tower" version = "0.5.2" @@ -1586,12 +1678,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - [[package]] name = "unicode-xid" version = "0.2.6" @@ -1754,28 +1840,6 @@ dependencies = [ "semver", ] -[[package]] -name = "wast" -version = "245.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cf1149285569120b8ce39db8b465e8a2b55c34cbb586bd977e43e2bc7300bf" -dependencies = [ - "bumpalo", - "leb128fmt", - "memchr", - "unicode-width", - "wasm-encoder", -] - -[[package]] -name = "wat" -version = "1.245.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd48d1679b6858988cb96b154dda0ec5bbb09275b71db46057be37332d5477be" -dependencies = [ - "wast", -] - [[package]] name = "web-sys" version = "0.3.83" @@ -1805,6 +1869,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81995fafaaaf6ae47a7d0cc83c67caf92aeb7e5331650ae6ff856f7c0c60c459" +dependencies = [ + "libc", +] + [[package]] name = "winapi-util" version = "0.1.11" @@ -2071,6 +2144,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" + [[package]] name = "wit-bindgen" version = "0.46.0" @@ -2079,8 +2158,8 @@ checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "wit-bindgen-core" -version = "0.53.1" -source = "git+https://github.com/bytecodealliance/wit-bindgen?rev=3ee9fe20a5bce398360d5d291e81a4224a6d7c76#3ee9fe20a5bce398360d5d291e81a4224a6d7c76" +version = "0.54.0" +source = "git+https://github.com/dicej/wit-bindgen?rev=661ade1e#661ade1ee006000b0cc5fe066c4d08ff1bca3cef" dependencies = [ "anyhow", "heck", @@ -2089,8 +2168,8 @@ dependencies = [ [[package]] name = "wit-bindgen-go" -version = "0.53.1" -source = "git+https://github.com/bytecodealliance/wit-bindgen?rev=3ee9fe20a5bce398360d5d291e81a4224a6d7c76#3ee9fe20a5bce398360d5d291e81a4224a6d7c76" +version = "0.54.0" +source = "git+https://github.com/dicej/wit-bindgen?rev=661ade1e#661ade1ee006000b0cc5fe066c4d08ff1bca3cef" dependencies = [ "anyhow", "heck", diff --git a/Cargo.toml b/Cargo.toml index 5c03e24..e3f9e2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,10 @@ members = ["./tests"] [workspace.dependencies] anyhow = "1.0.102" +bzip2 = "0.6.1" +once_cell = "1.21.3" +reqwest = { version = "0.13.1", features = ["blocking"] } +tar = "0.4" [workspace.package] version = "0.2.0" @@ -34,10 +38,18 @@ unnecessary_cast = 'warn' allow_attributes_without_reason = 'warn' [dependencies] -anyhow = { workspace = true} +anyhow = { workspace = true } +bzip2 = { workspace = true } +reqwest = { workspace = true } +tar = { workspace = true } clap = { version = "4.5.60", features = ["derive"] } regex = "1.12.3" -wat = { version = "1.245.1"} -wit-bindgen-go = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "3ee9fe20a5bce398360d5d291e81a4224a6d7c76" } +serde = { version = "1.0.228", features = ["derive"] } +toml = "1.1.0" +# TODO: Switch back to upstream once +# https://github.com/bytecodealliance/wit-bindgen/pull/1572 is merged: +wit-bindgen-go = { git = "https://github.com/dicej/wit-bindgen", rev = "661ade1e" } wit-component = "0.245.1" wit-parser = "0.245.1" +which = "8.0.2" +dirs = "6.0.0" diff --git a/src/cmd_bindings.rs b/src/cmd_bindings.rs index 8fa6b54..5a5560b 100644 --- a/src/cmd_bindings.rs +++ b/src/cmd_bindings.rs @@ -1,19 +1,19 @@ -use crate::utils::{make_path_absolute, parse_wit}; +use crate::utils::make_path_absolute; use anyhow::Result; use std::path::{Path, PathBuf}; +use wit_parser::{Resolve, WorldId}; #[allow(clippy::too_many_arguments)] pub fn generate_bindings( - wit_path: &[impl AsRef], - world: Option<&str>, - features: &[String], - all_features: bool, + resolve: &mut Resolve, + world: WorldId, generate_stubs: bool, should_format: bool, output: Option<&Path>, pkg_name: Option, + export_pkg_name: Option, + include_versions: bool, ) -> Result<()> { - let (mut resolve, world) = parse_wit(wit_path, world, features, all_features)?; let mut files = Default::default(); let format = if should_format { @@ -36,13 +36,15 @@ pub fn generate_bindings( generate_stubs, format, pkg_name, + export_pkg_name, + include_versions, ..Default::default() } .build() - .generate(&mut resolve, world, &mut files)?; + .generate(resolve, world, &mut files)?; let output_path = match output { - Some(p) => make_path_absolute(&p.to_path_buf())?, + Some(p) => make_path_absolute(p)?, None => PathBuf::from("."), }; diff --git a/src/cmd_build.rs b/src/cmd_build.rs index 156cdcf..f9ffdb3 100644 --- a/src/cmd_build.rs +++ b/src/cmd_build.rs @@ -1,23 +1,16 @@ use crate::utils::{check_go_version, make_path_absolute}; use anyhow::{Result, anyhow}; -use std::{path::PathBuf, process::Command}; +use std::{ + path::{Path, PathBuf}, + process::Command, +}; /// Compiles a Go application to a wasm module with `go build`. /// /// If the module is not going to be adapted to the component model, /// set the `only_wasip1` arg to true. -pub fn build_module( - go_module: Option<&PathBuf>, - out: Option<&PathBuf>, - go_path: Option<&PathBuf>, - only_wasip1: bool, -) -> Result { - let go = match &go_path { - Some(p) => make_path_absolute(p)?, - None => PathBuf::from("go"), - }; - - check_go_version(&go)?; +pub fn build_module(out: Option<&PathBuf>, go: &Path, only_wasip1: bool) -> Result { + check_go_version(go)?; let out_path_buf = match &out { Some(p) => make_path_absolute(p)?, @@ -33,22 +26,11 @@ pub fn build_module( .to_str() .ok_or_else(|| anyhow!("Output path is not valid unicode"))?; - let module_path = match &go_module { - Some(p) => { - if !p.is_dir() { - return Err(anyhow!("Module path '{}' is not a directory", p.display())); - } - p.to_str() - .ok_or_else(|| anyhow!("Module path is not valid unicode"))? - } - None => ".", - }; - // The -buildmode flag mutes the module's output, so it is ommitted let module_args = [ "build", "-C", - module_path, + ".", "-ldflags=-checklinkname=0", "-o", out_path, @@ -57,7 +39,7 @@ pub fn build_module( let component_args = [ "build", "-C", - module_path, + ".", "-buildmode=c-shared", "-ldflags=-checklinkname=0", "-o", @@ -65,13 +47,13 @@ pub fn build_module( ]; let output = if only_wasip1 { - Command::new(&go) + Command::new(go) .args(module_args) .env("GOOS", "wasip1") .env("GOARCH", "wasm") .output()? } else { - Command::new(&go) + Command::new(go) .args(component_args) .env("GOOS", "wasip1") .env("GOARCH", "wasm") diff --git a/src/cmd_test.rs b/src/cmd_test.rs index 7b47f50..1cd1954 100644 --- a/src/cmd_test.rs +++ b/src/cmd_test.rs @@ -12,15 +12,10 @@ use std::{ pub fn build_test_module( path: &Path, output_dir: Option<&PathBuf>, - go_path: Option<&PathBuf>, + go: &Path, only_wasip1: bool, ) -> Result { - let go = match &go_path { - Some(p) => make_path_absolute(p)?, - None => PathBuf::from("go"), - }; - - check_go_version(&go)?; + check_go_version(go)?; let test_wasm_path = { // The directory in which the test component will be placed @@ -68,7 +63,7 @@ pub fn build_test_module( ]; let output = if only_wasip1 { - Command::new(&go) + Command::new(go) .args(module_args) .env("GOOS", "wasip1") .env("GOARCH", "wasm") @@ -80,7 +75,7 @@ pub fn build_test_module( // TODO: for when we figure out how wasip2 tests are to be run #[allow(unreachable_code)] - Command::new(&go) + Command::new(go) .args(component_args) .env("GOOS", "wasip1") .env("GOARCH", "wasm") diff --git a/src/command.rs b/src/command.rs index f50dadf..336e2e5 100644 --- a/src/command.rs +++ b/src/command.rs @@ -2,7 +2,7 @@ use crate::{ cmd_bindings::generate_bindings, cmd_build::build_module, cmd_test::build_test_module, - utils::{embed_wit, module_to_component}, + utils::{dummy_wit, embed_wit, module_to_component, parse_wit, pick_go}, }; use anyhow::{Result, anyhow}; use clap::{Parser, Subcommand}; @@ -28,12 +28,30 @@ pub struct WitOpts { /// /// These paths can be either directories containing `*.wit` files, `*.wit` /// files themselves, or `*.wasm` files which are wasm-encoded WIT packages. + /// + /// Note that, unless `--ignore-toml-files` is specified, `componentize-go` + /// will also use `go list` to scan the current Go module and its + /// dependencies to find any `componentize-go.toml` files. The WIT + /// documents referenced by any such files will be added to this list + /// automatically. #[arg(long, short = 'd')] pub wit_path: Vec, - /// Name of world to target (or default world if `None`). + /// Name of world to target (or default world if not specified). + /// + /// This may be specified more than once, in which case the worlds will be + /// merged. + /// + /// Note that, unless `--ignore-toml-files` _or_ at least one `--world` + /// option is specified, `componentize-go` will use `go list` to scan the + /// current Go module and its dependencies to find any + /// `componentize-go.toml` files, and the WIT worlds referenced by any such + /// files will be used. #[arg(long, short = 'w')] - pub world: Option, + pub world: Vec, + + #[arg(long)] + pub ignore_toml_files: bool, /// Whether or not to activate all WIT features when processing WIT files. /// @@ -73,11 +91,14 @@ pub struct Build { #[arg(long, short = 'o')] pub output: Option, - /// The directory containing the "go.mod" file (or current directory if `None`). - #[arg(long = "mod")] - pub mod_path: Option, - /// The path to the Go binary (or look for binary in PATH if `None`). + /// + /// If the target WIT world uses async features, and the specified Go binary + /// (or the one in PATH if `None`) does not include [this + /// patch](https://github.com/golang/go/pull/76775), a patched version will + /// be downloaded, stored in the current user's [cache + /// directory](https://docs.rs/dirs/latest/dirs/fn.cache_dir.html), and used + /// for building. #[arg(long)] pub go: Option, @@ -111,6 +132,13 @@ pub struct Test { pub output: Option, /// The path to the Go binary (or look for binary in PATH if `None`). + /// + /// If the target WIT world uses async features, and the specified Go binary + /// (or the one in PATH if `None`) does not include [this + /// patch](https://github.com/golang/go/pull/76775), a patched version will + /// be downloaded, stored in the current user's [cache + /// directory](https://docs.rs/dirs/latest/dirs/fn.cache_dir.html), and used + /// for testing. #[arg(long)] pub go: Option, @@ -139,6 +167,24 @@ pub struct Bindings { /// otherwise (if None), the bindings will be organized for use as a standalone executable. #[arg(long)] pub pkg_name: Option, + + /// When `--pkg-name` is specified, optionally specify a different package + /// for exports. + /// + /// This allows you to put the exports and imports in separate packages when + /// building a library. If only `--pkg-name` is specified, this will + /// default to that value. + #[arg(long, requires = "pkg_name")] + pub export_pkg_name: Option, + + /// When generating Go package names, include the WIT package version even + /// if only one version of that package is referenced by the specified + /// world. + /// + /// By default, the version will only be included in the name if the world + /// references more than one version of the WIT package. + #[arg(long)] + pub include_versions: bool, } pub fn run + Clone, I: IntoIterator>(args: I) -> Result<()> { @@ -151,52 +197,63 @@ pub fn run + Clone, I: IntoIterator>(args: I) -> Res } fn build(wit_opts: WitOpts, build: Build) -> Result<()> { + let (resolve, world) = if build.wasip1 { + dummy_wit() + } else { + parse_wit( + &wit_opts.wit_path, + &wit_opts.world, + wit_opts.ignore_toml_files, + &wit_opts.features, + wit_opts.all_features, + )? + }; + + let go = &pick_go(&resolve, world, build.go.as_deref())?; + // Build a wasm module using `go build`. - let module = build_module( - build.mod_path.as_ref(), - build.output.as_ref(), - build.go.as_ref(), - build.wasip1, - )?; + let module = build_module(build.output.as_ref(), go, build.wasip1)?; if !build.wasip1 { // Embed the WIT documents in the wasip1 component. - embed_wit( - &module, - &wit_opts.wit_path, - wit_opts.world.as_deref(), - &wit_opts.features, - wit_opts.all_features, - )?; + embed_wit(&module, &resolve, world)?; // Update the wasm module to use the current component model ABI. - module_to_component(&module, &build.adapt)?; + module_to_component(&module, build.adapt.as_deref())?; } Ok(()) } fn test(wit_opts: WitOpts, test: Test) -> Result<()> { + let (resolve, world) = if test.wasip1 { + dummy_wit() + } else { + parse_wit( + &wit_opts.wit_path, + &wit_opts.world, + wit_opts.ignore_toml_files, + &wit_opts.features, + wit_opts.all_features, + )? + }; + + let go = &pick_go(&resolve, world, test.go.as_deref())?; + if test.pkg.is_empty() { return Err(anyhow!("Path to a package containing Go tests is required")); } for pkg in test.pkg.iter() { // Build a wasm module using `go test -c`. - let module = build_test_module(pkg, test.output.as_ref(), test.go.as_ref(), test.wasip1)?; + let module = build_test_module(pkg, test.output.as_ref(), go, test.wasip1)?; if !test.wasip1 { // Embed the WIT documents in the wasm module. - embed_wit( - &module, - &wit_opts.wit_path, - wit_opts.world.as_deref(), - &wit_opts.features, - wit_opts.all_features, - )?; + embed_wit(&module, &resolve, world)?; // Update the wasm module to use the current component model ABI. - module_to_component(&module, &test.adapt)?; + module_to_component(&module, test.adapt.as_deref())?; } } @@ -204,14 +261,22 @@ fn test(wit_opts: WitOpts, test: Test) -> Result<()> { } fn bindings(wit_opts: WitOpts, bindings: Bindings) -> Result<()> { - generate_bindings( - wit_opts.wit_path.as_ref(), - wit_opts.world.as_deref(), + let (mut resolve, world) = parse_wit( + &wit_opts.wit_path, + &wit_opts.world, + wit_opts.ignore_toml_files, &wit_opts.features, wit_opts.all_features, + )?; + + generate_bindings( + &mut resolve, + world, bindings.generate_stubs, bindings.format, bindings.output.as_deref(), bindings.pkg_name, + bindings.export_pkg_name, + bindings.include_versions, ) } diff --git a/src/utils.rs b/src/utils.rs index 7e8a03d..288b7ff 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,9 +1,33 @@ -use anyhow::{Context, Result, anyhow}; +use anyhow::{Context, Result, anyhow, bail}; +use bzip2::read::BzDecoder; +use serde::Deserialize; use std::{ + collections::BTreeSet, + fs::{self, File}, + io::Cursor, path::{Path, PathBuf}, process::Command, }; -use wit_parser::{PackageId, Resolve, WorldId}; +use tar::Archive; +use wit_parser::{ + CloneMaps, Function, Interface, Package, PackageId, PackageName, Resolve, Stability, Type, + TypeDef, TypeDefKind, World, WorldId, WorldItem, +}; + +pub fn dummy_wit() -> (Resolve, WorldId) { + let mut resolve = Resolve::default(); + let world = resolve.worlds.alloc(World { + name: "dummy-world".into(), + imports: Default::default(), + exports: Default::default(), + package: Default::default(), + docs: Default::default(), + stability: Default::default(), + includes: Default::default(), + span: Default::default(), + }); + (resolve, world) +} // In the rare case the snapshot needs to be updated, the latest version // can be found here: https://github.com/bytecodealliance/wasmtime/releases @@ -11,15 +35,18 @@ const WASIP1_SNAPSHOT_ADAPT: &[u8] = include_bytes!("wasi_snapshot_preview1.reac pub fn parse_wit( paths: &[impl AsRef], - world: Option<&str>, + worlds: &[String], + ignore_toml_files: bool, features: &[String], all_features: bool, ) -> Result<(Resolve, WorldId)> { + let (paths, worlds) = &maybe_add_dependencies(paths, worlds, ignore_toml_files)?; + // If no WIT directory was provided as a parameter and none were referenced // by Go packages, use ./wit by default. if paths.is_empty() { let paths = &[Path::new("wit")]; - return parse_wit(paths, world, features, all_features); + return parse_wit(paths, worlds, ignore_toml_files, features, all_features); } debug_assert!(!paths.is_empty(), "The paths should not be empty"); @@ -38,17 +65,111 @@ pub fn parse_wit( } let mut main_packages: Vec = vec![]; - for path in paths.iter().map(AsRef::as_ref) { + for path in paths.iter() { let (pkg, _files) = resolve.push_path(path)?; main_packages.push(pkg); } - let world = resolve.select_world(&main_packages, world)?; + let world = match &worlds[..] { + [] => resolve.select_world(&main_packages, None)?, + [world] => resolve.select_world(&main_packages, Some(world))?, + worlds => { + let worlds = worlds + .iter() + .map(|world| resolve.select_world(&main_packages, Some(world))) + .collect::>>()?; + + let union_package = resolve.packages.alloc(Package { + name: PackageName { + namespace: "componentize-go".into(), + name: "union".into(), + version: None, + }, + docs: Default::default(), + interfaces: Default::default(), + worlds: Default::default(), + }); + + let union_world = resolve.worlds.alloc(World { + name: "union".into(), + imports: Default::default(), + exports: Default::default(), + package: Some(union_package), + docs: Default::default(), + stability: Stability::Unknown, + includes: Default::default(), + span: Default::default(), + }); + + resolve.packages[union_package] + .worlds + .insert("union".into(), union_world); + + for &world in &worlds { + resolve.merge_worlds(world, union_world, &mut CloneMaps::default())?; + } + + union_world + } + }; + Ok((resolve, world)) } +/// Unless `ignore_toml_files` is `true`, use `go list` to search the current +/// module and its dependencies for any `componentize-go.toml` files. The WIT +/// path and/or world specified in each such file will be added to the +/// respective list and returned. +fn maybe_add_dependencies( + paths: &[impl AsRef], + worlds: &[String], + ignore_toml_files: bool, +) -> Result<(Vec, Vec)> { + let mut paths = paths + .iter() + .map(|v| PathBuf::from(v.as_ref())) + .collect::>(); + let mut worlds = worlds.iter().cloned().collect::>(); + // Only add worlds from `componentize-go.toml` files if none were specified + // explicitly via the CLI: + let add_worlds = worlds.is_empty(); + + if !ignore_toml_files && Path::new("go.mod").exists() { + let output = Command::new("go") + .args(["list", "-mod=readonly", "-m", "-f", "{{.Dir}}", "all"]) + .output()?; + if !output.status.success() { + bail!( + "`go list` failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[derive(Deserialize)] + struct ComponentizeGoConfig { + #[serde(default)] + worlds: Vec, + #[serde(default)] + wit_paths: Vec, + } + + for module in String::from_utf8(output.stdout)?.lines() { + let module = PathBuf::from(module); + if let Ok(manifest) = fs::read_to_string(module.join("componentize-go.toml")) { + let config = toml::from_str::(&manifest)?; + if add_worlds { + worlds.extend(config.worlds); + } + paths.extend(config.wit_paths.into_iter().map(|v| module.join(v))); + } + } + } + + Ok((paths.into_iter().collect(), worlds.into_iter().collect())) +} + // Converts a relative path to an absolute path. -pub fn make_path_absolute(p: &PathBuf) -> Result { +pub fn make_path_absolute(p: &Path) -> Result { if p.is_relative() { Ok(std::env::current_dir()?.join(p)) } else { @@ -56,34 +177,26 @@ pub fn make_path_absolute(p: &PathBuf) -> Result { } } -pub fn embed_wit( - wasm_file: &PathBuf, - wit_path: &[PathBuf], - world: Option<&str>, - features: &[String], - all_features: bool, -) -> Result<()> { - let mut wasm = wat::Parser::new().parse_file(wasm_file)?; - let (resolve, world_id) = parse_wit(wit_path, world, features, all_features)?; +pub fn embed_wit(wasm_file: &Path, resolve: &Resolve, world: WorldId) -> Result<()> { + let mut wasm = fs::read(wasm_file)?; wit_component::embed_component_metadata( &mut wasm, - &resolve, - world_id, + resolve, + world, wit_component::StringEncoding::UTF8, )?; - std::fs::write(wasm_file, wasm) - .context(format!("failed to write '{}'", wasm_file.display()))?; + fs::write(wasm_file, wasm).context(format!("failed to write '{}'", wasm_file.display()))?; Ok(()) } /// Update the wasm module to use the current component model ABI. -pub fn module_to_component(wasm_file: &PathBuf, adapt_file: &Option) -> Result<()> { - let wasm: Vec = wat::Parser::new().parse_file(wasm_file)?; +pub fn module_to_component(wasm_file: &Path, adapt_file: Option<&Path>) -> Result<()> { + let wasm: Vec = fs::read(wasm_file)?; let mut encoder = wit_component::ComponentEncoder::default().validate(true); encoder = encoder.module(&wasm)?; let adapt_bytes = if let Some(adapt) = adapt_file { - std::fs::read(adapt) + fs::read(adapt) .with_context(|| format!("failed to read adapt file '{}'", adapt.display()))? } else { WASIP1_SNAPSHOT_ADAPT.to_vec() @@ -94,14 +207,13 @@ pub fn module_to_component(wasm_file: &PathBuf, adapt_file: &Option) -> .encode() .context("failed to encode component from module")?; - std::fs::write(wasm_file, bytes) - .context(format!("failed to write `{}`", wasm_file.display()))?; + fs::write(wasm_file, bytes).context(format!("failed to write `{}`", wasm_file.display()))?; Ok(()) } /// Ensure that the Go version is compatible with the embedded Wasm tooling. -pub fn check_go_version(go_path: &PathBuf) -> Result<()> { +pub fn check_go_version(go_path: &Path) -> Result<()> { let output = Command::new(go_path).arg("version").output()?; if !output.status.success() { @@ -140,3 +252,184 @@ pub fn check_go_version(go_path: &PathBuf) -> Result<()> { )) } } + +fn check_go_async_support(go: &Path) -> Option<()> { + fs::read_to_string( + go.parent()? + .parent()? + .join("src") + .join("runtime") + .join("lock_wasip1.go"), + ) + .ok()? + .contains("wasiOnIdle") + .then_some(()) +} + +fn world_needs_async(resolve: &Resolve, world: WorldId) -> bool { + fn typedef_needs_async(resolve: &Resolve, ty: &TypeDef) -> bool { + match &ty.kind { + TypeDefKind::Record(v) => v.fields.iter().any(|v| type_needs_async(resolve, v.ty)), + TypeDefKind::Tuple(v) => v.types.iter().any(|&v| type_needs_async(resolve, v)), + TypeDefKind::Variant(v) => v + .cases + .iter() + .any(|v| v.ty.map(|v| type_needs_async(resolve, v)).unwrap_or(false)), + &TypeDefKind::Type(v) + | &TypeDefKind::Option(v) + | &TypeDefKind::List(v) + | &TypeDefKind::FixedLengthList(v, _) => type_needs_async(resolve, v), + TypeDefKind::Result(v) => { + v.ok.map(|v| type_needs_async(resolve, v)).unwrap_or(false) + || v.err.map(|v| type_needs_async(resolve, v)).unwrap_or(false) + } + &TypeDefKind::Map(k, v) => type_needs_async(resolve, k) || type_needs_async(resolve, v), + TypeDefKind::Future(_) | TypeDefKind::Stream(_) => true, + TypeDefKind::Resource + | TypeDefKind::Handle(_) + | TypeDefKind::Flags(_) + | TypeDefKind::Enum(_) => false, + TypeDefKind::Unknown => unreachable!(), + } + } + + fn type_needs_async(resolve: &Resolve, ty: Type) -> bool { + match ty { + Type::Bool + | Type::U8 + | Type::U16 + | Type::U32 + | Type::U64 + | Type::S8 + | Type::S16 + | Type::S32 + | Type::S64 + | Type::F32 + | Type::F64 + | Type::Char + | Type::String + | Type::ErrorContext => false, + Type::Id(id) => typedef_needs_async(resolve, &resolve.types[id]), + } + } + + let function_needs_async = |fun: &Function| { + fun.kind.is_async() + || fun.params.iter().any(|v| type_needs_async(resolve, v.ty)) + || fun + .result + .map(|ty| type_needs_async(resolve, ty)) + .unwrap_or(false) + }; + + let interface_needs_async = |interface: &Interface| { + interface + .types + .values() + .any(|&id| type_needs_async(resolve, Type::Id(id))) + || interface.functions.values().any(function_needs_async) + }; + + let world = &resolve.worlds[world]; + world + .imports + .values() + .chain(world.exports.values()) + .any(|item| { + match item { + &WorldItem::Interface { id, .. } => { + if interface_needs_async(&resolve.interfaces[id]) { + return true; + } + } + WorldItem::Function(fun) => { + if function_needs_async(fun) { + return true; + } + } + &WorldItem::Type { id, .. } => { + if type_needs_async(resolve, Type::Id(id)) { + return true; + } + } + } + false + }) +} + +pub fn pick_go(resolve: &Resolve, world: WorldId, go_path: Option<&Path>) -> Result { + let go = match go_path { + Some(p) => Some(make_path_absolute(p)?), + None => which::which("go").ok(), + }; + + if let Some(go) = go { + if check_go_version(&go).is_err() { + eprintln!( + "Note: {} is not a compatible version of Go; will use downloaded version.", + go.display() + ); + } else if world_needs_async(resolve, world) && check_go_async_support(&go).is_none() { + eprintln!( + "Note: {} does not support async operation; will use downloaded version.\n\ + See https://github.com/golang/go/pull/76775 for details.", + go.display() + ); + } else { + return Ok(go); + } + } else { + eprintln!("Note: `go` command not found; will use downloaded version."); + } + + let Some(cache_dir) = dirs::cache_dir() else { + bail!("unable to determine cache directory for current user"); + }; + + // Determine OS and architecture + let os = match std::env::consts::OS { + "macos" => "darwin", + "linux" => "linux", + "windows" => "windows", + bad_os => panic!("OS not supported: {bad_os}"), + }; + + // Map to Go's naming conventions + let arch = match std::env::consts::ARCH { + "aarch64" => "arm64", + "x86_64" => "amd64", + bad_arch => panic!("ARCH not supported: {bad_arch}"), + }; + + let cache_dir = &cache_dir.join("componentize-go"); + let name = &format!("go-{os}-{arch}-bootstrap"); + let dir = cache_dir.join(name); + let bin = dir.join("bin").join("go"); + + fs::create_dir_all(cache_dir)?; + + // Grab a lock to avoid concurrent downloads + let lock_file = File::create(cache_dir.join("lock"))?; + lock_file.lock()?; + + if !bin.exists() { + let url = format!( + "https://github.com/dicej/go/releases/download/go1.25.5-wasi-on-idle/{name}.tbz" + ); + + eprintln!("Downloading patched Go from {url}."); + + let content = reqwest::blocking::get(&url)?.error_for_status()?.bytes()?; + + eprintln!("Extracting patched Go to {}.", cache_dir.display()); + + Archive::new(BzDecoder::new(Cursor::new(content))).unpack(cache_dir)?; + } + + check_go_version(&bin)?; + check_go_async_support(&bin).ok_or_else(|| anyhow!("downloaded Go does not support async"))?; + + eprintln!("Using {}.", bin.display()); + + Ok(bin) +} diff --git a/tests/Cargo.toml b/tests/Cargo.toml index e25713e..6340b4b 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -9,8 +9,8 @@ workspace = true [dependencies] componentize-go = { path = ".." } anyhow = { workspace = true } -bzip2 = "0.6.1" +bzip2 = { workspace = true } once_cell = "1.21.3" -reqwest = { version = "0.13.1", features = ["blocking"] } -tar = "0.4" +reqwest = { workspace = true } +tar = { workspace = true } tokio = { version = "1.49.0", features = ["macros", "rt-multi-thread"] } diff --git a/tests/src/lib.rs b/tests/src/lib.rs index b7ed602..f9bd7c8 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -30,68 +30,6 @@ mod tests { root_manifest.join("target/release/componentize-go") }); - // TODO: Once the patch is merged in Big Go, this needs to be removed. - async fn patched_go_path() -> PathBuf { - let test_manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let root_manifest = test_manifest.parent().unwrap(); - - // Determine OS and architecture - let os = match std::env::consts::OS { - "macos" => "darwin", - "linux" => "linux", - "windows" => "windows", - bad_os => panic!("OS not supported: {bad_os}"), - }; - - // Map to Go's naming conventions - let arch = match std::env::consts::ARCH { - "aarch64" => "arm64", - "x86_64" => "amd64", - bad_arch => panic!("ARCH not supported: {bad_arch}"), - }; - - let go_dir = format!("go-{os}-{arch}-bootstrap"); - let go_path = root_manifest.join(&go_dir); - let go_bin = go_path.join("bin").join("go"); - - // Skip if already installed - if go_bin.exists() { - return go_bin; - } - - // Download the patched Go toolchain - let archive_name = format!("{go_dir}.tbz"); - let archive_path = root_manifest.join(&archive_name); - let download_url = format!( - "https://github.com/dicej/go/releases/download/go1.25.5-wasi-on-idle/{archive_name}" - ); - - println!("Downloading patched Go from {download_url}"); - let response = reqwest::get(&download_url) - .await - .expect("Failed to download patched Go"); - - std::fs::write( - &archive_path, - response.bytes().await.expect("Failed to read download"), - ) - .expect("Failed to write archive"); - - // Extract the archive - println!("Extracting {} to {}", archive_name, root_manifest.display()); - let tar_file = std::fs::File::open(&archive_path).expect("Failed to open archive"); - let tar_decoder = bzip2::read::BzDecoder::new(tar_file); - let mut archive = tar::Archive::new(tar_decoder); - archive - .unpack(root_manifest) - .expect("Failed to extract archive"); - - // Clean up archive - std::fs::remove_file(&archive_path).ok(); - - go_bin - } - struct App { /// The path to the example application path: String, @@ -159,10 +97,10 @@ mod tests { } } - fn build_test_modules(&self, go: Option<&PathBuf>) -> Result<()> { + fn build_test_modules(&self) -> Result<()> { let test_pkgs = self.tests.as_ref().expect("missing test_pkg_paths"); - self.generate_bindings(go)?; + self.generate_bindings()?; let mut test_cmd = Command::new(COMPONENTIZE_GO_PATH.as_path()); test_cmd @@ -262,7 +200,7 @@ mod tests { Ok(()) } - fn build_module(&self, go: Option<&PathBuf>) -> Result<()> { + fn build_module(&self) -> Result<()> { // Build component let mut build_cmd = Command::new(COMPONENTIZE_GO_PATH.as_path()); build_cmd @@ -270,10 +208,6 @@ mod tests { .arg("--wasip1") .args(["-o", &self.wasm_path]); - if let Some(go_path) = go.as_ref() { - build_cmd.args(["--go", go_path.to_str().unwrap()]); - } - // Run `go build` in the same directory as the go.mod file. build_cmd.current_dir(&self.path); @@ -293,8 +227,8 @@ mod tests { Ok(()) } - fn build_component(&self, go: Option<&PathBuf>) -> Result<()> { - self.generate_bindings(go)?; + fn build_component(&self) -> Result<()> { + self.generate_bindings()?; // Build component let mut build_cmd = Command::new(COMPONENTIZE_GO_PATH.as_path()); @@ -304,10 +238,6 @@ mod tests { .arg("build") .args(["-o", &self.wasm_path]); - if let Some(go_path) = go.as_ref() { - build_cmd.args(["--go", go_path.to_str().unwrap()]); - } - // Run `go build` in the same directory as the go.mod file. build_cmd.current_dir(&self.path); @@ -327,7 +257,7 @@ mod tests { Ok(()) } - fn generate_bindings(&self, go: Option<&PathBuf>) -> Result<()> { + fn generate_bindings(&self) -> Result<()> { let bindings_output = Command::new(COMPONENTIZE_GO_PATH.as_path()) .args(["-w", self.world.as_ref().expect("missing WIT world")]) .args(["-d", self.wit_path.as_ref().expect("missing WIT path")]) @@ -347,17 +277,12 @@ mod tests { } // Tidy Go mod - let tidy_output = Command::new(if let Some(path) = go.as_ref() { - String::from(path.to_str().unwrap()) - } else { - // Default to PATH - "go".to_string() - }) - .arg("mod") - .arg("tidy") - .current_dir(&self.path) - .output() - .expect("failed to tidy Go mod"); + let tidy_output = Command::new("go") + .arg("mod") + .arg("tidy") + .current_dir(&self.path) + .output() + .expect("failed to tidy Go mod"); if !tidy_output.status.success() { return Err(anyhow!("{}", String::from_utf8_lossy(&tidy_output.stderr))); } @@ -415,7 +340,7 @@ mod tests { #[test] fn example_wasip1() { let app = App::new("../examples/wasip1", None, None, false); - app.build_module(None).expect("failed to build app module"); + app.build_module().expect("failed to build app module"); app.run_module().expect("failed to run app module"); } @@ -439,13 +364,13 @@ mod tests { true, ); - app.build_component(None).expect("failed to build app"); + app.build_component().expect("failed to build app"); app.run_component("/", "Hello, world!") .await .expect("app failed to run"); - app.build_test_modules(None) + app.build_test_modules() .expect("failed to build app unit tests"); app.run_test_modules() @@ -455,8 +380,7 @@ mod tests { #[tokio::test] async fn example_wasip3() { let mut app = App::new("../examples/wasip3", Some("wasip3-example"), None, true); - app.build_component(Some(&patched_go_path().await)) - .expect("failed to build app"); + app.build_component().expect("failed to build app"); app.run_component("/hello", "Hello, world!") .await .expect("app failed to run");