1// Small script used to calculate the matrix of tests that are going to be
2// performed for a CI run.
3//
4// This is invoked by the `determine` step and is written in JS because I
5// couldn't figure out how to write it in bash.
6
7const fs = require('fs');
8const { spawn } = require('node:child_process');
9
10// Number of generic buckets to shard crates into. Note that we additionally add
11// single-crate buckets for our biggest crates.
12const GENERIC_BUCKETS = 3;
13
14// Crates which are their own buckets. These are the very slowest to
15// compile-and-test crates.
16const SINGLE_CRATE_BUCKETS = ["wasmtime", "wasmtime-cli", "wasmtime-wasi"];
17
18const ubuntu = 'ubuntu-24.04';
19const windows = 'windows-2025';
20const macos = 'macos-15';
21
22// This is the small, fast-to-execute matrix we use for PRs before they enter
23// the merge queue. Same schema as `FULL_MATRIX`.
24const FAST_MATRIX = [
25  {
26    "os": ubuntu,
27    "name": "Test Linux x86_64",
28    "filter": "linux-x64",
29    "isa": "x64",
30  },
31];
32
33// This is the full, unsharded, and unfiltered matrix of what we test on
34// CI. This includes a number of platforms and a number of cross-compiled
35// targets that are emulated with QEMU. This must be kept tightly in sync with
36// the `test` step in `main.yml`.
37//
38// The supported keys here are:
39//
40// * `os` - the github-actions name of the runner os
41//
42// * `name` - the human-readable name of the job
43//
44// * `filter` - a string which if `prtest:$filter` is in the commit messages
45//   it'll force running this test suite on PR CI.
46//
47// * `isa` - changes to `cranelift/codegen/src/$isa` will automatically run this
48//   test suite.
49//
50// * `target` - used for cross-compiles if present. Effectively Cargo's
51//   `--target` option for all its operations.
52//
53// * `gcc_package`, `gcc`, `qemu`, `qemu_target` - configuration for building
54//   QEMU and installing cross compilers to execute a cross-compiled test suite
55//   on CI.
56//
57// * `rust` - the Rust version to install, and if unset this'll be set to
58//   `default`
59const FULL_MATRIX = [
60  ...FAST_MATRIX,
61  {
62    "os": ubuntu,
63    "name": "Test MSRV on Linux x86_64",
64    "filter": "linux-x64",
65    "isa": "x64",
66    "rust": "msrv",
67  },
68  {
69    "os": ubuntu,
70    "name": "Test Linux x86_64 with MPK",
71    "filter": "linux-x64",
72    "isa": "x64"
73  },
74  {
75    "os": ubuntu,
76    "name": "Test Linux x86_64 with ASAN",
77    "filter": "asan",
78    "rust": "wasmtime-ci-pinned-nightly",
79    "target": "x86_64-unknown-linux-gnu",
80  },
81  {
82    "os": macos,
83    "name": "Test macOS x86_64",
84    "filter": "macos-x64",
85    "target": "x86_64-apple-darwin",
86  },
87  {
88    "os": macos,
89    "name": "Test macOS arm64",
90    "filter": "macos-arm64",
91    "target": "aarch64-apple-darwin",
92  },
93  {
94    "os": windows,
95    "name": "Test Windows MSVC x86_64",
96    "filter": "windows-x64",
97  },
98  {
99    "os": windows,
100    "target": "x86_64-pc-windows-gnu",
101    "name": "Test Windows MinGW x86_64",
102    "filter": "mingw-x64"
103  },
104  {
105    "os": ubuntu + '-arm',
106    "target": "aarch64-unknown-linux-gnu",
107    "name": "Test Linux arm64",
108    "filter": "linux-arm64",
109    "isa": "aarch64",
110  },
111  {
112    "os": ubuntu,
113    "target": "s390x-unknown-linux-gnu",
114    "gcc_package": "gcc-s390x-linux-gnu",
115    "gcc": "s390x-linux-gnu-gcc",
116    "qemu": "qemu-s390x -L /usr/s390x-linux-gnu",
117    "qemu_target": "s390x-linux-user",
118    "name": "Test Linux s390x",
119    "filter": "linux-s390x",
120    "isa": "s390x"
121  },
122  {
123    "os": ubuntu,
124    "target": "riscv64gc-unknown-linux-gnu",
125    "gcc_package": "gcc-riscv64-linux-gnu",
126    "gcc": "riscv64-linux-gnu-gcc",
127    "qemu": "qemu-riscv64 -cpu rv64,v=true,vlen=256,vext_spec=v1.0,zfa=true,zfh=true,zba=true,zbb=true,zbc=true,zbs=true,zbkb=true,zcb=true,zicond=true,zvfh=true -L /usr/riscv64-linux-gnu",
128    "qemu_target": "riscv64-linux-user",
129    "name": "Test Linux riscv64",
130    "filter": "linux-riscv64",
131    "isa": "riscv64",
132  },
133  {
134    "name": "Tests on i686-unknown-linux-gnu",
135    "os": ubuntu,
136    "target": "i686-unknown-linux-gnu",
137    "gcc_package": "gcc-i686-linux-gnu",
138    "gcc": "i686-linux-gnu-gcc",
139  },
140  {
141    "name": "Tests on armv7-unknown-linux-gnueabihf",
142    "os": ubuntu,
143    "target": "armv7-unknown-linux-gnueabihf",
144    "gcc_package": "gcc-arm-linux-gnueabihf",
145    "gcc": "arm-linux-gnueabihf-gcc",
146    "qemu": "qemu-arm -L /usr/arm-linux-gnueabihf -E LD_LIBRARY_PATH=/usr/arm-linux-gnueabihf/lib",
147    "qemu_target": "arm-linux-user",
148  },
149];
150
151/// Get the workspace's full list of member crates.
152async function getWorkspaceMembers() {
153  // Spawn a `cargo metadata` subprocess, accumulate its JSON output from
154  // `stdout`, and wait for it to exit.
155  const child = spawn("cargo", ["metadata"], { encoding: "utf8" });
156  let data = "";
157  child.stdout.on("data", chunk => data += chunk);
158  await new Promise((resolve, reject) => {
159    child.on("close", resolve);
160    child.on("error", reject);
161  });
162
163  // Get the names of the crates in the workspace from the JSON metadata by
164  // building a package-id to name map and then translating the package-ids
165  // listed as workspace members.
166  const metadata = JSON.parse(data);
167  const id_to_name = {};
168  for (const pkg of metadata.packages) {
169    id_to_name[pkg.id] = pkg.name;
170  }
171  return metadata.workspace_members.map(m => id_to_name[m]);
172}
173
174/// For each given target configuration, shard the workspace's crates into
175/// buckets across that config.
176///
177/// This is essentially a `flat_map` where each config that logically tests all
178/// crates in the workspace is mapped to N sharded configs that each test only a
179/// subset of crates in the workspace. Each sharded config's subset of crates to
180/// test are disjoint from all its siblings, and the union of all these siblings'
181/// crates to test is the full workspace members set.
182///
183/// With some poetic license around a `crates_to_test` key that doesn't actually
184/// exist, logically each element of the input `configs` list gets transformed
185/// like this:
186///
187///     { os: "ubuntu-latest", isa: "x64", ..., crates: "all" }
188///
189///     ==>
190///
191///     [
192///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime"] },
193///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime-cli"] },
194///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime-wasi"] },
195///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["cranelift", "cranelift-codegen", ...] },
196///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime-slab", "cranelift-entity", ...] },
197///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["cranelift-environ", "wasmtime-cli-flags", ...] },
198///       ...
199///     ]
200///
201/// Note that `crates: "all"` is implicit in the input and omitted. Similarly,
202/// `crates: [...]` in each output config is actually implemented via adding a
203/// `bucket` key, which contains the CLI flags we must pass to `cargo` to run
204/// tests for just this config's subset of crates.
205async function shard(configs) {
206  const members = await getWorkspaceMembers();
207
208  // Divide the workspace crates into N disjoint subsets. Crates that are
209  // particularly expensive to compile and test form their own singleton subset.
210  const buckets = Array.from({ length: GENERIC_BUCKETS }, _ => new Set());
211  let i = 0;
212  for (const crate of members) {
213    if (SINGLE_CRATE_BUCKETS.indexOf(crate) != -1) continue;
214    buckets[i].add(crate);
215    i = (i + 1) % GENERIC_BUCKETS;
216  }
217  for (crate of SINGLE_CRATE_BUCKETS) {
218    buckets.push(new Set([crate]));
219  }
220
221  // For each config, expand it into N configs, one for each disjoint set we
222  // created above.
223  const sharded = [];
224  for (const config of configs) {
225    for (const bucket of buckets) {
226      sharded.push(Object.assign(
227        {},
228        config,
229        {
230          name: `${config.name} (${Array.from(bucket).join(', ')})`,
231          // We run tests via `cargo test --workspace`, so exclude crates that
232          // aren't in this bucket, rather than naming only the crates that are
233          // in this bucket.
234          bucket: members
235            .map(c => bucket.has(c) ? `--package ${c}` : `--exclude ${c}`)
236            .join(" "),
237        }
238      ));
239    }
240  }
241  return sharded;
242}
243
244async function main() {
245  // Our first argument is a file that is a giant json blob which contains at
246  // least all the messages for all of the commits that were a part of this PR.
247  // This is used to test if any commit message includes a string.
248  const commits = fs.readFileSync(process.argv[2]).toString();
249
250  // The second argument is a file that contains the names of all files modified
251  // for a PR, used for file-based filters.
252  const names = fs.readFileSync(process.argv[3]).toString();
253
254  for (let config of FULL_MATRIX) {
255    if (config.rust === undefined) {
256      config.rust = 'default';
257    }
258  }
259
260  // If the optional third argument to this script is `true` then that means all
261  // tests are being run and no filtering should happen.
262  if (process.argv[4] == 'true') {
263    console.log(JSON.stringify(await shard(FULL_MATRIX), undefined, 2));
264    return;
265  }
266
267  // When we aren't running the full CI matrix, filter configs down to just the
268  // relevant bits based on files changed in this commit or if the commit asks
269  // for a certain config to run.
270  const filtered = FULL_MATRIX.filter(config => {
271    // If an ISA-specific test was modified, then include that ISA config.
272    if (config.isa && names.includes(`cranelift/codegen/src/isa/${config.isa}`)) {
273      return true;
274    }
275
276    // If any runtest was modified, include all ISA configs as runtests can
277    // target any backend.
278    if (names.includes(`cranelift/filetests/filetests/runtests`)) {
279      if (config.isa !== undefined)
280        return true;
281    }
282
283    // If the commit explicitly asks for this test config, then include it.
284    if (config.filter && commits.includes(`prtest:${config.filter}`)) {
285      return true;
286    }
287
288    return false;
289  });
290
291  // If at least one test is being run via our filters then run those tests.
292  if (filtered.length > 0) {
293    console.log(JSON.stringify(await shard(filtered), undefined, 2));
294    return;
295  }
296
297  // Otherwise if nothing else is being run, run the fast subset of the matrix.
298  console.log(JSON.stringify(await shard(FAST_MATRIX), undefined, 2));
299}
300
301main()
302