1 //! Run the tests in `wasi_testsuite` using Wasmtime's CLI binary and checking
2 //! the results with a [wasi-testsuite] spec.
3 //!
4 //! [wasi-testsuite]: https://github.com/WebAssembly/wasi-testsuite
5
6 use libtest_mimic::{Arguments, Trial};
7 use serde_derive::Deserialize;
8 use std::collections::HashMap;
9 use std::fmt::Write;
10 use std::fs;
11 use std::path::Path;
12 use std::process::Output;
13 use tempfile::TempDir;
14 use wasmtime::{Result, ToWasmtimeResult as _, format_err};
15 use wit_component::ComponentEncoder;
16
17 const KNOWN_FAILURES: &[&str] = &[
18 "filesystem-hard-links",
19 "filesystem-read-directory",
20 // FIXME(#11524)
21 "remove_directory_trailing_slashes",
22 #[cfg(target_vendor = "apple")]
23 "filesystem-advise",
24 // FIXME(WebAssembly/wasi-testsuite#128)
25 #[cfg(windows)]
26 "fd_fdstat_set_rights",
27 #[cfg(windows)]
28 "filesystem-flags-and-type",
29 #[cfg(windows)]
30 "path_link",
31 #[cfg(windows)]
32 "dangling_fd",
33 #[cfg(windows)]
34 "dangling_symlink",
35 #[cfg(windows)]
36 "file_allocate",
37 #[cfg(windows)]
38 "file_pread_pwrite",
39 #[cfg(windows)]
40 "file_seek_tell",
41 #[cfg(windows)]
42 "file_truncation",
43 #[cfg(windows)]
44 "file_unbuffered_write",
45 #[cfg(windows)]
46 "interesting_paths",
47 #[cfg(windows)]
48 "isatty",
49 #[cfg(windows)]
50 "fd_readdir",
51 #[cfg(windows)]
52 "nofollow_errors",
53 #[cfg(windows)]
54 "overwrite_preopen",
55 #[cfg(windows)]
56 "path_exists",
57 #[cfg(windows)]
58 "path_filestat",
59 #[cfg(windows)]
60 "path_open_create_existing",
61 #[cfg(windows)]
62 "path_open_dirfd_not_dir",
63 #[cfg(windows)]
64 "path_open_missing",
65 #[cfg(windows)]
66 "path_open_read_write",
67 #[cfg(windows)]
68 "path_rename",
69 #[cfg(windows)]
70 "path_rename_dir_trailing_slashes",
71 #[cfg(windows)]
72 "path_symlink_trailing_slashes",
73 #[cfg(windows)]
74 "readlink",
75 #[cfg(windows)]
76 "remove_nonempty_directory",
77 #[cfg(windows)]
78 "renumber",
79 #[cfg(windows)]
80 "symlink_create",
81 #[cfg(windows)]
82 "stdio",
83 #[cfg(windows)]
84 "symlink_filestat",
85 #[cfg(windows)]
86 "truncation_rights",
87 #[cfg(windows)]
88 "symlink_loop",
89 #[cfg(windows)]
90 "unlink_file_trailing_slashes",
91 // Once cm-async changes have percolated this can be removed.
92 "filesystem-flags-and-type",
93 "multi-clock-wait",
94 "monotonic-clock",
95 "filesystem-advise",
96 // Wasmtime's snapshot of WASIp3 APIs is different than what these tests are
97 // expecting.
98 "wall-clock",
99 "http-response",
100 ];
101
main() -> Result<()>102 fn main() -> Result<()> {
103 env_logger::init();
104
105 let mut trials = Vec::new();
106 if !cfg!(miri) {
107 find_tests("tests/wasi_testsuite/wasi-common".as_ref(), &mut trials).unwrap();
108 find_tests("tests/wasi_testsuite/wasi-threads".as_ref(), &mut trials).unwrap();
109 }
110
111 libtest_mimic::run(&Arguments::from_args(), trials).exit()
112 }
113
find_tests(path: &Path, trials: &mut Vec<Trial>) -> Result<()>114 fn find_tests(path: &Path, trials: &mut Vec<Trial>) -> Result<()> {
115 for entry in path.read_dir()? {
116 let entry = entry?;
117 let path = entry.path();
118 if entry.file_type()?.is_dir() {
119 find_tests(&path, trials)?;
120 continue;
121 }
122 if path.extension().and_then(|s| s.to_str()) != Some("wasm") {
123 continue;
124 }
125
126 // Test the core wasm itself.
127 trials.push(Trial::test(
128 format!("wasmtime-wasi - {}", path.display()),
129 {
130 let path = path.clone();
131 move || run_test(&path, false).map_err(|e| format!("{e:?}").into())
132 },
133 ));
134
135 // Also test the component version using the wasip1 adapter. Note that
136 // this is skipped for `wasi-threads` since that's not supported in
137 // components and it's also skipped for assemblyscript because that
138 // doesn't support the wasip1 adapter.
139 if !path.iter().any(|p| p == "wasm32-wasip3")
140 && !path.iter().any(|p| p == "wasi-threads")
141 && !path.iter().any(|p| p == "assemblyscript")
142 {
143 trials.push(Trial::test(
144 format!("wasip1 adapter - {}", path.display()),
145 move || run_test(&path, true).map_err(|e| format!("{e:?}").into()),
146 ));
147 }
148 }
149 Ok(())
150 }
151
run_test(path: &Path, componentize: bool) -> Result<()>152 fn run_test(path: &Path, componentize: bool) -> Result<()> {
153 let wasmtime = Path::new(env!("CARGO_BIN_EXE_wasmtime"));
154 let test_name = path.file_stem().unwrap().to_str().unwrap();
155 let target_dir = wasmtime.parent().unwrap().parent().unwrap();
156 let parent_dir = path.parent().ok_or(format_err!("module has no parent?"))?;
157 let spec = if let Ok(contents) = fs::read_to_string(&path.with_extension("json")) {
158 serde_json::from_str(&contents)?
159 } else {
160 Spec::default()
161 };
162
163 let mut td = TempDir::new_in(&target_dir)?;
164 td.disable_cleanup(true);
165 let path = if componentize {
166 let module = fs::read(path).expect("read wasm module");
167 let component = ComponentEncoder::default()
168 .module(module.as_slice())
169 .to_wasmtime_result()?
170 .validate(true)
171 .adapter(
172 "wasi_snapshot_preview1",
173 &fs::read(test_programs_artifacts::ADAPTER_COMMAND)?,
174 )
175 .to_wasmtime_result()?
176 .encode()
177 .to_wasmtime_result()?;
178 let stem = path.file_stem().unwrap().to_str().unwrap();
179 let component_path = td.path().join(format!("{stem}.component.wasm"));
180 fs::write(&component_path, component)?;
181 component_path
182 } else {
183 path.to_path_buf()
184 };
185
186 let Spec {
187 args,
188 dirs,
189 env,
190 exit_code: _,
191 stderr: _,
192 stdout: _,
193 } = &spec;
194 let mut cmd = wasmtime_test_util::command(wasmtime);
195 cmd.arg("run");
196 for dir in dirs {
197 cmd.arg("--dir");
198 let src = parent_dir.join(dir);
199 let dst = td.path().join(dir);
200 cp_r(&src, &dst)?;
201 cmd.arg(format!("{}::{dir}", dst.display()));
202 }
203 for (k, v) in env {
204 cmd.arg("--env");
205 cmd.arg(format!("{k}={v}"));
206 }
207 let mut should_fail = KNOWN_FAILURES.contains(&test_name);
208 if path.iter().any(|p| p == "wasm32-wasip3") {
209 cmd.arg("-Sp3,http").arg("-Wcomponent-model-async");
210 if !cfg!(feature = "component-model-async") {
211 should_fail = true;
212 }
213 }
214 cmd.arg(path);
215 cmd.args(args);
216
217 let result = cmd.output()?;
218 td.disable_cleanup(true);
219 let ok = spec == result;
220 match (ok, should_fail) {
221 // If this test passed and is not a known failure, or if it failed and
222 // it's a known failure, then flag this test as "ok".
223 (true, false) | (false, true) => Ok(()),
224
225 // If this test failed and it's not known to fail, explain why.
226 (false, false) => {
227 td.disable_cleanup(false);
228 let mut msg = String::new();
229 writeln!(msg, " command: {cmd:?}")?;
230 writeln!(msg, " spec: {spec:#?}")?;
231 writeln!(msg, " result.status: {}", result.status)?;
232 if !result.stdout.is_empty() {
233 write!(
234 msg,
235 " result.stdout:\n {}",
236 String::from_utf8_lossy(&result.stdout).replace("\n", "\n ")
237 )?;
238 }
239 if !result.stderr.is_empty() {
240 writeln!(
241 msg,
242 " result.stderr:\n {}",
243 String::from_utf8_lossy(&result.stderr).replace("\n", "\n ")
244 )?;
245 }
246 wasmtime::bail!("{msg}\nFAILED! The result does not match the specification");
247 }
248
249 // If this test passed but it's flagged as should be failed, then fail
250 // this test for someone to update `KNOWN_FAILURES`.
251 (true, true) => {
252 wasmtime::bail!("test passed but it's listed in `KNOWN_FAILURES`")
253 }
254 }
255 }
256
cp_r(path: &Path, dst: &Path) -> Result<()>257 fn cp_r(path: &Path, dst: &Path) -> Result<()> {
258 fs::create_dir(dst)?;
259 for entry in path.read_dir()? {
260 let entry = entry?;
261 let path = entry.path();
262 let dst = dst.join(entry.file_name());
263 if entry.file_type()?.is_dir() {
264 cp_r(&path, &dst)?;
265 } else {
266 fs::copy(&path, &dst)?;
267 }
268 }
269 Ok(())
270 }
271
272 #[derive(Debug, Default, Deserialize)]
273 struct Spec {
274 #[serde(default)]
275 args: Vec<String>,
276 #[serde(default)]
277 dirs: Vec<String>,
278 #[serde(default)]
279 env: HashMap<String, String>,
280 exit_code: Option<i32>,
281 stderr: Option<String>,
282 stdout: Option<String>,
283 }
284
285 impl PartialEq<Output> for Spec {
eq(&self, other: &Output) -> bool286 fn eq(&self, other: &Output) -> bool {
287 self.exit_code.unwrap_or(0) == other.status.code().unwrap()
288 && matches_or_missing(&self.stdout, &other.stdout)
289 && matches_or_missing(&self.stderr, &other.stderr)
290 }
291 }
292
matches_or_missing(a: &Option<String>, b: &[u8]) -> bool293 fn matches_or_missing(a: &Option<String>, b: &[u8]) -> bool {
294 a.as_ref()
295 .map(|s| s == &String::from_utf8_lossy(b))
296 .unwrap_or(true)
297 }
298