1 use heck::*; 2 use std::collections::{BTreeMap, HashSet}; 3 use std::env; 4 use std::fs; 5 use std::path::{Path, PathBuf}; 6 use std::process::Command; 7 use wit_component::ComponentEncoder; 8 9 fn main() { 10 let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); 11 12 Artifacts { 13 out_dir, 14 deps: HashSet::default(), 15 } 16 .build(); 17 } 18 19 struct Artifacts { 20 out_dir: PathBuf, 21 deps: HashSet<String>, 22 } 23 24 struct Test { 25 /// Not all tests can be built at build-time, for example C/C++ tests require 26 /// the `WASI_SDK_PATH` environment variable which isn't available on all 27 /// machines. The `Option` here encapsulates tests that were not able to be 28 /// built. 29 /// 30 /// For tests that were not able to be built their error is deferred to 31 /// test-time when the test is actually run. For C/C++ tests this means that 32 /// only when running debuginfo tests does the error show up, for example. 33 core_wasm: Option<PathBuf>, 34 35 name: String, 36 } 37 38 impl Artifacts { 39 fn build(&mut self) { 40 let mut generated_code = String::new(); 41 // Build adapters used below for componentization. 42 let reactor_adapter = self.build_adapter(&mut generated_code, "reactor", &[]); 43 let command_adapter = self.build_adapter( 44 &mut generated_code, 45 "command", 46 &["--no-default-features", "--features=command"], 47 ); 48 let proxy_adapter = self.build_adapter( 49 &mut generated_code, 50 "proxy", 51 &["--no-default-features", "--features=proxy"], 52 ); 53 54 // Build all test programs both in Rust and C/C++. 55 let mut tests = Vec::new(); 56 self.build_rust_tests(&mut tests); 57 self.build_non_rust_tests(&mut tests); 58 59 // With all our `tests` now compiled generate various macos for each 60 // test along with constants pointing to various paths. Note that 61 // components are created here as well from core modules. 62 let mut kinds = BTreeMap::new(); 63 let missing_sdk_path = 64 PathBuf::from("Asset not compiled, WASI_SDK_PATH missing at compile time"); 65 for test in tests.iter() { 66 let shouty_snake = test.name.to_shouty_snake_case(); 67 let snake = test.name.to_snake_case(); 68 69 let core_wasm = test.core_wasm.as_deref().unwrap_or(&missing_sdk_path); 70 generated_code += 71 &format!("pub const {shouty_snake}: &'static str = {core_wasm:?};\n",); 72 generated_code += &format!( 73 "#[macro_export] macro_rules! {snake}_bytes {{ 74 () => {{ include_bytes!({core_wasm:?}) }} 75 }}", 76 ); 77 78 // Bucket, based on the name of the test, into a "kind" which 79 // generates a `foreach_*` macro below. 80 let kind = match test.name.as_str() { 81 s if s.starts_with("p1_cli_") 82 || s.starts_with("p2_cli_") 83 || s.starts_with("p3_cli_") => 84 { 85 "cli" 86 } 87 s if s.starts_with("p1_") => "p1", 88 s if s.starts_with("p2_http_") => "p2_http", 89 s if s.starts_with("p2_api_") => "p2_api", 90 s if s.starts_with("p2_") => "p2", 91 s if s.starts_with("p3_http_") => "p3_http", 92 s if s.starts_with("p3_api_") => "p3_api", 93 s if s.starts_with("p3_") => "p3", 94 s if s.starts_with("nn_") => "nn", 95 s if s.starts_with("piped_") => "piped", 96 s if s.starts_with("dwarf_") => "dwarf", 97 s if s.starts_with("config_") => "config", 98 s if s.starts_with("keyvalue_") => "keyvalue", 99 s if s.starts_with("tls_") => "tls", 100 s if s.starts_with("async_") => "async", 101 s if s.starts_with("fuzz_") => "fuzz", 102 // If you're reading this because you hit this panic, either add 103 // it to a test suite above or add a new "suite". The purpose of 104 // the categorization above is to have a static assertion that 105 // tests added are actually run somewhere, so as long as you're 106 // also adding test code somewhere that's ok. 107 other => { 108 panic!("don't know how to classify test name `{other}` to a kind") 109 } 110 }; 111 if !kind.is_empty() { 112 kinds.entry(kind).or_insert(Vec::new()).push(&test.name); 113 } 114 115 // Generate a component from each test. 116 if test.name == "dwarf_imported_memory" 117 || test.name == "dwarf_shared_memory" 118 || test.name.starts_with("nn_witx") 119 { 120 continue; 121 } 122 let adapter = match test.name.as_str() { 123 "reactor" => &reactor_adapter, 124 s if s.starts_with("p3_") => &reactor_adapter, 125 s if s.starts_with("p2_api_proxy") => &proxy_adapter, 126 _ => &command_adapter, 127 }; 128 let path = match &test.core_wasm { 129 Some(path) => self.compile_component(path, adapter), 130 None => missing_sdk_path.clone(), 131 }; 132 generated_code += 133 &format!("pub const {shouty_snake}_COMPONENT: &'static str = {path:?};\n"); 134 generated_code += &format!( 135 "#[macro_export] macro_rules! {snake}_component_bytes {{ 136 () => {{ include_bytes!({path:?}) }} 137 }}", 138 ); 139 } 140 141 for (kind, targets) in kinds { 142 generated_code += &format!("#[macro_export]"); 143 generated_code += &format!("macro_rules! foreach_{kind} {{\n"); 144 generated_code += &format!(" ($mac:ident) => {{\n"); 145 for target in targets { 146 generated_code += &format!("$mac!({target});\n") 147 } 148 generated_code += &format!(" }}\n"); 149 generated_code += &format!("}}\n"); 150 } 151 152 std::fs::write(self.out_dir.join("gen.rs"), generated_code).unwrap(); 153 } 154 155 fn build_rust_tests(&mut self, tests: &mut Vec<Test>) { 156 println!("cargo:rerun-if-env-changed=MIRI_TEST_CWASM_DIR"); 157 let release_mode = env::var_os("MIRI_TEST_CWASM_DIR").is_some(); 158 159 let mut cmd = cargo(); 160 cmd.arg("build"); 161 if release_mode { 162 cmd.arg("--release"); 163 } 164 cmd.arg("--target=wasm32-wasip1") 165 .arg("--package=test-programs") 166 .env("CARGO_TARGET_DIR", &self.out_dir) 167 .env("CARGO_PROFILE_DEV_DEBUG", "2") 168 .env("RUSTFLAGS", rustflags()) 169 .env_remove("CARGO_ENCODED_RUSTFLAGS"); 170 eprintln!("running: {cmd:?}"); 171 let status = cmd.status().unwrap(); 172 assert!(status.success()); 173 174 let meta = cargo_metadata::MetadataCommand::new().exec().unwrap(); 175 let targets = meta 176 .packages 177 .iter() 178 .find(|p| p.name == "test-programs") 179 .unwrap() 180 .targets 181 .iter() 182 .filter(move |t| t.kind == &[cargo_metadata::TargetKind::Bin]) 183 .map(|t| &t.name) 184 .collect::<Vec<_>>(); 185 186 for target in targets { 187 let wasm = self 188 .out_dir 189 .join("wasm32-wasip1") 190 .join(if release_mode { "release" } else { "debug" }) 191 .join(format!("{target}.wasm")); 192 self.read_deps_of(&wasm); 193 tests.push(Test { 194 core_wasm: Some(wasm), 195 name: target.to_string(), 196 }) 197 } 198 } 199 200 // Build the WASI Preview 1 adapter, and get the binary: 201 fn build_adapter( 202 &mut self, 203 generated_code: &mut String, 204 name: &str, 205 features: &[&str], 206 ) -> Vec<u8> { 207 let mut cmd = cargo(); 208 cmd.arg("build") 209 .arg("--release") 210 .arg("--package=wasi-preview1-component-adapter") 211 .arg("--target=wasm32-unknown-unknown") 212 .env("CARGO_TARGET_DIR", &self.out_dir) 213 .env("RUSTFLAGS", rustflags()) 214 .env_remove("CARGO_ENCODED_RUSTFLAGS"); 215 for f in features { 216 cmd.arg(f); 217 } 218 eprintln!("running: {cmd:?}"); 219 let status = cmd.status().unwrap(); 220 assert!(status.success()); 221 222 let artifact = self 223 .out_dir 224 .join("wasm32-unknown-unknown") 225 .join("release") 226 .join("wasi_snapshot_preview1.wasm"); 227 let adapter = self 228 .out_dir 229 .join(format!("wasi_snapshot_preview1.{name}.wasm")); 230 std::fs::copy(&artifact, &adapter).unwrap(); 231 self.read_deps_of(&artifact); 232 println!("wasi {name} adapter: {:?}", &adapter); 233 generated_code.push_str(&format!( 234 "pub const ADAPTER_{}: &'static str = {adapter:?};\n", 235 name.to_shouty_snake_case(), 236 )); 237 fs::read(&adapter).unwrap() 238 } 239 240 // Compile a component, return the path of the binary: 241 fn compile_component(&self, wasm: &Path, adapter: &[u8]) -> PathBuf { 242 println!("creating a component from {wasm:?}"); 243 let module = fs::read(wasm).expect("read wasm module"); 244 let component = ComponentEncoder::default() 245 .module(module.as_slice()) 246 .unwrap() 247 .validate(true) 248 .adapter("wasi_snapshot_preview1", adapter) 249 .unwrap() 250 .encode() 251 .expect("module can be translated to a component"); 252 let out_dir = wasm.parent().unwrap(); 253 let stem = wasm.file_stem().unwrap().to_str().unwrap(); 254 let component_path = out_dir.join(format!("{stem}.component.wasm")); 255 fs::write(&component_path, component).expect("write component to disk"); 256 component_path 257 } 258 259 fn build_non_rust_tests(&mut self, tests: &mut Vec<Test>) { 260 const ASSETS_REL_SRC_DIR: &'static str = "../src/bin"; 261 println!("cargo:rerun-if-changed={ASSETS_REL_SRC_DIR}"); 262 263 for entry in fs::read_dir(ASSETS_REL_SRC_DIR).unwrap() { 264 let entry = entry.unwrap(); 265 let path = entry.path(); 266 let name = path.file_stem().unwrap().to_str().unwrap().to_owned(); 267 match path.extension().and_then(|s| s.to_str()) { 268 // Compile C/C++ tests with clang 269 Some("c") | Some("cc") => self.build_c_or_cpp_test(path, name, tests), 270 271 // just a header, part of another test. 272 Some("h") => {} 273 274 // Convert the text format to binary and use it as a test. 275 Some("wat") => { 276 let wasm = wat::parse_file(&path).unwrap(); 277 let core_wasm = self.out_dir.join(&name).with_extension("wasm"); 278 fs::write(&core_wasm, &wasm).unwrap(); 279 tests.push(Test { 280 name, 281 core_wasm: Some(core_wasm), 282 }); 283 } 284 285 // these are built above in `build_rust_tests` 286 Some("rs") => {} 287 288 // Prevent stray files for now that we don't understand. 289 Some(_) => panic!("unknown file extension on {path:?}"), 290 291 None => unreachable!("no extension in path {path:?}"), 292 } 293 } 294 } 295 296 fn build_c_or_cpp_test(&mut self, path: PathBuf, name: String, tests: &mut Vec<Test>) { 297 println!("compiling {path:?}"); 298 println!("cargo:rerun-if-changed={}", path.display()); 299 let contents = std::fs::read_to_string(&path).unwrap(); 300 let config = 301 wasmtime_test_util::wast::parse_test_config::<CTestConfig>(&contents, "//!").unwrap(); 302 303 if config.skip { 304 return; 305 } 306 307 // The debug tests relying on these assets are ignored by default, 308 // so we cannot force the requirement of having a working WASI SDK 309 // install on everyone. At the same time, those tests (due to their 310 // monolithic nature), are always compiled, so we still have to 311 // produce the path constants. To solve this, we move the failure 312 // of missing WASI SDK from compile time to runtime by producing 313 // fake paths (that themselves will serve as diagnostic messages). 314 let wasi_sdk_path = match env::var_os("WASI_SDK_PATH") { 315 Some(path) => PathBuf::from(path), 316 None => { 317 tests.push(Test { 318 name, 319 core_wasm: None, 320 }); 321 return; 322 } 323 }; 324 325 let wasm_path = self.out_dir.join(&name).with_extension("wasm"); 326 327 let mut cmd = Command::new(wasi_sdk_path.join("bin/wasm32-wasip1-clang")); 328 cmd.arg(&path); 329 for file in config.extra_files.iter() { 330 cmd.arg(path.parent().unwrap().join(file)); 331 } 332 cmd.arg("-g"); 333 cmd.args(&config.flags); 334 cmd.arg("-o"); 335 cmd.arg(&wasm_path); 336 // If optimizations are enabled, clang will look for wasm-opt in PATH 337 // and run it. This will strip DWARF debug info, which we don't want. 338 cmd.env("PATH", ""); 339 println!("running: {cmd:?}"); 340 let result = cmd.status().expect("failed to spawn clang"); 341 assert!(result.success()); 342 343 if config.dwp { 344 let mut dwp = Command::new(wasi_sdk_path.join("bin/llvm-dwp")); 345 dwp.arg("-e") 346 .arg(&wasm_path) 347 .arg("-o") 348 .arg(self.out_dir.join(&name).with_extension("dwp")); 349 assert!(dwp.status().expect("failed to spawn llvm-dwp").success()); 350 } 351 352 tests.push(Test { 353 name, 354 core_wasm: Some(wasm_path), 355 }); 356 } 357 358 /// Helper function to read the `*.d` file that corresponds to `artifact`, an 359 /// artifact of a Cargo compilation. 360 /// 361 /// This function will "parse" the makefile-based dep-info format to learn about 362 /// what files each binary depended on to ensure that this build script reruns 363 /// if any of these files change. 364 /// 365 /// See 366 /// <https://doc.rust-lang.org/nightly/cargo/reference/build-cache.html#dep-info-files> 367 /// for more info. 368 fn read_deps_of(&mut self, artifact: &Path) { 369 let deps_file = artifact.with_extension("d"); 370 let contents = std::fs::read_to_string(&deps_file).expect("failed to read deps file"); 371 for line in contents.lines() { 372 let Some(pos) = line.find(": ") else { 373 continue; 374 }; 375 let line = &line[pos + 2..]; 376 let mut parts = line.split_whitespace(); 377 while let Some(part) = parts.next() { 378 let mut file = part.to_string(); 379 while file.ends_with('\\') { 380 file.pop(); 381 file.push(' '); 382 file.push_str(parts.next().unwrap()); 383 } 384 if !self.deps.contains(&file) { 385 println!("cargo:rerun-if-changed={file}"); 386 self.deps.insert(file); 387 } 388 } 389 } 390 } 391 } 392 393 #[derive(serde_derive::Deserialize)] 394 #[serde(deny_unknown_fields, rename_all = "kebab-case")] 395 struct CTestConfig { 396 #[serde(default)] 397 flags: Vec<String>, 398 #[serde(default)] 399 extra_files: Vec<String>, 400 #[serde(default)] 401 dwp: bool, 402 #[serde(default)] 403 skip: bool, 404 } 405 406 fn cargo() -> Command { 407 // Miri configures its own sysroot which we don't want to use, so remove 408 // miri's own wrappers around rustc to ensure that we're using the real 409 // rustc to build these programs. 410 let mut cargo = Command::new("cargo"); 411 if std::env::var("CARGO_CFG_MIRI").is_ok() { 412 cargo.env_remove("RUSTC").env_remove("RUSTC_WRAPPER"); 413 } 414 cargo 415 } 416 417 fn rustflags() -> &'static str { 418 match option_env!("RUSTFLAGS") { 419 // If we're in CI which is denying warnings then deny warnings to code 420 // built here too to keep the tree warning-free. 421 Some(s) if s.contains("-D warnings") => "-D warnings", 422 _ => "", 423 } 424 } 425