use heck::*; use std::collections::{BTreeMap, HashSet}; use std::env; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use wit_component::ComponentEncoder; fn main() { let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); Artifacts { out_dir, deps: HashSet::default(), } .build(); } struct Artifacts { out_dir: PathBuf, deps: HashSet, } struct Test { /// Not all tests can be built at build-time, for example C/C++ tests require /// the `WASI_SDK_PATH` environment variable which isn't available on all /// machines. The `Option` here encapsulates tests that were not able to be /// built. /// /// For tests that were not able to be built their error is deferred to /// test-time when the test is actually run. For C/C++ tests this means that /// only when running debuginfo tests does the error show up, for example. core_wasm: Option, name: String, } impl Artifacts { fn build(&mut self) { let mut generated_code = String::new(); // Build adapters used below for componentization. let reactor_adapter = self.build_adapter(&mut generated_code, "reactor", &[]); let command_adapter = self.build_adapter( &mut generated_code, "command", &["--no-default-features", "--features=command"], ); let proxy_adapter = self.build_adapter( &mut generated_code, "proxy", &["--no-default-features", "--features=proxy"], ); // Build all test programs both in Rust and C/C++. let mut tests = Vec::new(); self.build_rust_tests(&mut tests); self.build_non_rust_tests(&mut tests); // With all our `tests` now compiled generate various macos for each // test along with constants pointing to various paths. Note that // components are created here as well from core modules. let mut kinds = BTreeMap::new(); let missing_sdk_path = PathBuf::from("Asset not compiled, WASI_SDK_PATH missing at compile time"); for test in tests.iter() { let shouty_snake = test.name.to_shouty_snake_case(); let snake = test.name.to_snake_case(); let core_wasm = test.core_wasm.as_deref().unwrap_or(&missing_sdk_path); generated_code += &format!("pub const {shouty_snake}: &'static str = {core_wasm:?};\n",); generated_code += &format!( "#[macro_export] macro_rules! {snake}_bytes {{ () => {{ include_bytes!({core_wasm:?}) }} }}", ); // Bucket, based on the name of the test, into a "kind" which // generates a `foreach_*` macro below. let kind = match test.name.as_str() { s if s.starts_with("p1_cli_") || s.starts_with("p2_cli_") || s.starts_with("p3_cli_") => { "cli" } s if s.starts_with("p1_") => "p1", s if s.starts_with("p2_http_") => "p2_http", s if s.starts_with("p2_api_") => "p2_api", s if s.starts_with("p2_tls_") => "p2_tls", s if s.starts_with("p2_") => "p2", s if s.starts_with("p3_http_") => "p3_http", s if s.starts_with("p3_api_") => "p3_api", s if s.starts_with("p3_tls_") => "p3_tls", s if s.starts_with("p3_") => "p3", s if s.starts_with("nn_") => "nn", s if s.starts_with("piped_") => "piped", s if s.starts_with("debugger_") => "debugger", s if s.starts_with("guest_debug_") => "guest_debug", s if s.starts_with("dwarf_") => "dwarf", s if s.starts_with("config_") => "config", s if s.starts_with("keyvalue_") => "keyvalue", s if s.starts_with("async_") => "async", s if s.starts_with("fuzz_") => "fuzz", // If you're reading this because you hit this panic, either add // it to a test suite above or add a new "suite". The purpose of // the categorization above is to have a static assertion that // tests added are actually run somewhere, so as long as you're // also adding test code somewhere that's ok. other => { panic!("don't know how to classify test name `{other}` to a kind") } }; if !kind.is_empty() { kinds.entry(kind).or_insert(Vec::new()).push(&test.name); } // Generate a component from each test. if test.name == "dwarf_imported_memory" || test.name == "dwarf_shared_memory" || test.name.starts_with("nn_witx") { continue; } let adapter = match test.name.as_str() { "reactor" => &reactor_adapter, s if s.starts_with("p3_") => &reactor_adapter, s if s.starts_with("p2_api_proxy") => &proxy_adapter, _ => &command_adapter, }; let path = match &test.core_wasm { Some(path) => self.compile_component(path, adapter), None => missing_sdk_path.clone(), }; generated_code += &format!("pub const {shouty_snake}_COMPONENT: &'static str = {path:?};\n"); generated_code += &format!( "#[macro_export] macro_rules! {snake}_component_bytes {{ () => {{ include_bytes!({path:?}) }} }}", ); } for (kind, targets) in kinds { generated_code += &format!("#[macro_export]"); generated_code += &format!("macro_rules! foreach_{kind} {{\n"); generated_code += &format!(" ($mac:ident) => {{\n"); for target in targets { generated_code += &format!("$mac!({target});\n") } generated_code += &format!(" }}\n"); generated_code += &format!("}}\n"); } std::fs::write(self.out_dir.join("gen.rs"), generated_code).unwrap(); } fn build_rust_tests(&mut self, tests: &mut Vec) { println!("cargo:rerun-if-env-changed=MIRI_TEST_CWASM_DIR"); let release_mode = env::var_os("MIRI_TEST_CWASM_DIR").is_some(); let mut cmd = cargo(); cmd.arg("build"); if release_mode { cmd.arg("--release"); } cmd.arg("--target=wasm32-wasip1") .arg("--package=test-programs") .env("CARGO_TARGET_DIR", &self.out_dir) .env("CARGO_PROFILE_DEV_DEBUG", "2") .env("RUSTFLAGS", rustflags()) .env_remove("CARGO_ENCODED_RUSTFLAGS"); eprintln!("running: {cmd:?}"); let status = cmd.status().unwrap(); assert!(status.success()); let meta = cargo_metadata::MetadataCommand::new().exec().unwrap(); let targets = meta .packages .iter() .find(|p| p.name == "test-programs") .unwrap() .targets .iter() .filter(move |t| t.kind == &[cargo_metadata::TargetKind::Bin]) .map(|t| &t.name) .collect::>(); for target in targets { let wasm = self .out_dir .join("wasm32-wasip1") .join(if release_mode { "release" } else { "debug" }) .join(format!("{target}.wasm")); self.read_deps_of(&wasm); tests.push(Test { core_wasm: Some(wasm), name: target.to_string(), }) } } // Build the WASI Preview 1 adapter, and get the binary: fn build_adapter( &mut self, generated_code: &mut String, name: &str, features: &[&str], ) -> Vec { let mut cmd = cargo(); cmd.arg("build") .arg("--release") .arg("--package=wasi-preview1-component-adapter") .arg("--target=wasm32-unknown-unknown") .env("CARGO_TARGET_DIR", &self.out_dir) .env("RUSTFLAGS", rustflags()) .env_remove("CARGO_ENCODED_RUSTFLAGS"); for f in features { cmd.arg(f); } eprintln!("running: {cmd:?}"); let status = cmd.status().unwrap(); assert!(status.success()); let artifact = self .out_dir .join("wasm32-unknown-unknown") .join("release") .join("wasi_snapshot_preview1.wasm"); let adapter = self .out_dir .join(format!("wasi_snapshot_preview1.{name}.wasm")); std::fs::copy(&artifact, &adapter).unwrap(); self.read_deps_of(&artifact); println!("wasi {name} adapter: {:?}", &adapter); generated_code.push_str(&format!( "pub const ADAPTER_{}: &'static str = {adapter:?};\n", name.to_shouty_snake_case(), )); fs::read(&adapter).unwrap() } // Compile a component, return the path of the binary: fn compile_component(&self, wasm: &Path, adapter: &[u8]) -> PathBuf { println!("creating a component from {wasm:?}"); let module = fs::read(wasm).expect("read wasm module"); let component = ComponentEncoder::default() .module(module.as_slice()) .unwrap() .validate(true) .adapter("wasi_snapshot_preview1", adapter) .unwrap() .encode() .expect("module can be translated to a component"); let out_dir = wasm.parent().unwrap(); let stem = wasm.file_stem().unwrap().to_str().unwrap(); let component_path = out_dir.join(format!("{stem}.component.wasm")); fs::write(&component_path, component).expect("write component to disk"); component_path } fn build_non_rust_tests(&mut self, tests: &mut Vec) { const ASSETS_REL_SRC_DIR: &'static str = "../src/bin"; println!("cargo:rerun-if-changed={ASSETS_REL_SRC_DIR}"); for entry in fs::read_dir(ASSETS_REL_SRC_DIR).unwrap() { let entry = entry.unwrap(); let path = entry.path(); let name = path.file_stem().unwrap().to_str().unwrap().to_owned(); match path.extension().and_then(|s| s.to_str()) { // Compile C/C++ tests with clang Some("c") | Some("cc") => self.build_c_or_cpp_test(path, name, tests), // just a header, part of another test. Some("h") => {} // Convert the text format to binary and use it as a test. Some("wat") => { let wasm = wat::parse_file(&path).unwrap(); let core_wasm = self.out_dir.join(&name).with_extension("wasm"); fs::write(&core_wasm, &wasm).unwrap(); tests.push(Test { name, core_wasm: Some(core_wasm), }); } // these are built above in `build_rust_tests` Some("rs") => {} // Prevent stray files for now that we don't understand. Some(_) => panic!("unknown file extension on {path:?}"), None => unreachable!("no extension in path {path:?}"), } } } fn build_c_or_cpp_test(&mut self, path: PathBuf, name: String, tests: &mut Vec) { println!("compiling {path:?}"); println!("cargo:rerun-if-changed={}", path.display()); let contents = std::fs::read_to_string(&path).unwrap(); let config = wasmtime_test_util::wast::parse_test_config::(&contents, "//!").unwrap(); if config.skip { return; } // The debug tests relying on these assets are ignored by default, // so we cannot force the requirement of having a working WASI SDK // install on everyone. At the same time, those tests (due to their // monolithic nature), are always compiled, so we still have to // produce the path constants. To solve this, we move the failure // of missing WASI SDK from compile time to runtime by producing // fake paths (that themselves will serve as diagnostic messages). let wasi_sdk_path = match env::var_os("WASI_SDK_PATH") { Some(path) => PathBuf::from(path), None => { tests.push(Test { name, core_wasm: None, }); return; } }; let wasm_path = self.out_dir.join(&name).with_extension("wasm"); let mut cmd = Command::new(wasi_sdk_path.join("bin/wasm32-wasip1-clang")); cmd.arg(&path); for file in config.extra_files.iter() { cmd.arg(path.parent().unwrap().join(file)); } cmd.arg("-g"); cmd.args(&config.flags); cmd.arg("-o"); cmd.arg(&wasm_path); // If optimizations are enabled, clang will look for wasm-opt in PATH // and run it. This will strip DWARF debug info, which we don't want. cmd.env("PATH", ""); println!("running: {cmd:?}"); let result = cmd.status().expect("failed to spawn clang"); assert!(result.success()); if config.dwp { let mut dwp = Command::new(wasi_sdk_path.join("bin/llvm-dwp")); dwp.arg("-e") .arg(&wasm_path) .arg("-o") .arg(self.out_dir.join(&name).with_extension("dwp")); assert!(dwp.status().expect("failed to spawn llvm-dwp").success()); } tests.push(Test { name, core_wasm: Some(wasm_path), }); } /// Helper function to read the `*.d` file that corresponds to `artifact`, an /// artifact of a Cargo compilation. /// /// This function will "parse" the makefile-based dep-info format to learn about /// what files each binary depended on to ensure that this build script reruns /// if any of these files change. /// /// See /// /// for more info. fn read_deps_of(&mut self, artifact: &Path) { let deps_file = artifact.with_extension("d"); let contents = std::fs::read_to_string(&deps_file).expect("failed to read deps file"); for line in contents.lines() { let Some(pos) = line.find(": ") else { continue; }; let line = &line[pos + 2..]; let mut parts = line.split_whitespace(); while let Some(part) = parts.next() { let mut file = part.to_string(); while file.ends_with('\\') { file.pop(); file.push(' '); file.push_str(parts.next().unwrap()); } if !self.deps.contains(&file) { println!("cargo:rerun-if-changed={file}"); self.deps.insert(file); } } } } } #[derive(serde_derive::Deserialize)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct CTestConfig { #[serde(default)] flags: Vec, #[serde(default)] extra_files: Vec, #[serde(default)] dwp: bool, #[serde(default)] skip: bool, } fn cargo() -> Command { // Miri configures its own sysroot which we don't want to use, so remove // miri's own wrappers around rustc to ensure that we're using the real // rustc to build these programs. let mut cargo = Command::new("cargo"); if std::env::var("CARGO_CFG_MIRI").is_ok() { cargo.env_remove("RUSTC").env_remove("RUSTC_WRAPPER"); } cargo } fn rustflags() -> &'static str { match option_env!("RUSTFLAGS") { // If we're in CI which is denying warnings then deny warnings to code // built here too to keep the tree warning-free. Some(s) if s.contains("-D warnings") => "-D warnings", _ => "", } }