1// Small script used to calculate the matrix of tests that are going to be
2// performed for a CI run.
3//
4// This is invoked by the `determine` step and is written in JS because I
5// couldn't figure out how to write it in bash.
6
7const fs = require('fs');
8const { spawn } = require('node:child_process');
9
10// Number of generic buckets to shard crates into. Note that we additionally add
11// single-crate buckets for our biggest crates.
12const GENERIC_BUCKETS = 3;
13
14// Crates which are their own buckets. These are the very slowest to
15// compile-and-test crates.
16const SINGLE_CRATE_BUCKETS = ["wasmtime", "wasmtime-cli", "wasmtime-wasi"];
17
18const ubuntu = 'ubuntu-24.04';
19const windows = 'windows-2025';
20const macos = 'macos-15';
21
22// This is the small, fast-to-execute matrix we use for PRs before they enter
23// the merge queue. Same schema as `FULL_MATRIX`.
24const FAST_MATRIX = [
25  {
26    "name": "Test Linux x86_64",
27    "os": ubuntu,
28    "filter": "linux-x64",
29    "isa": "x64",
30  },
31];
32
33// This is the full, unsharded, and unfiltered matrix of what we test on
34// CI. This includes a number of platforms and a number of cross-compiled
35// targets that are emulated with QEMU. This must be kept tightly in sync with
36// the `test` step in `main.yml`.
37//
38// The supported keys here are:
39//
40// * `os` - the github-actions name of the runner os
41//
42// * `name` - the human-readable name of the job
43//
44// * `filter` - a string which if `prtest:$filter` is in the commit messages
45//   it'll force running this test suite on PR CI.
46//
47// * `isa` - changes to `cranelift/codegen/src/$isa` will automatically run this
48//   test suite.
49//
50// * `target` - used for cross-compiles if present. Effectively Cargo's
51//   `--target` option for all its operations.
52//
53// * `gcc_package`, `gcc`, `qemu`, `qemu_target` - configuration for building
54//   QEMU and installing cross compilers to execute a cross-compiled test suite
55//   on CI.
56//
57// * `sde` - if `true`, indicates this test should use Intel SDE for instruction
58//   emulation. SDE will be set up and configured as the test runner.
59//
60// * `rust` - the Rust version to install, and if unset this'll be set to
61//   `default`
62const FULL_MATRIX = [
63  ...FAST_MATRIX,
64  {
65    "name": "Test MSRV",
66    "os": ubuntu,
67    "filter": "linux-x64",
68    "isa": "x64",
69    "rust": "msrv",
70  },
71  {
72    "name": "Test MPK",
73    "os": ubuntu,
74    "filter": "linux-x64",
75    "isa": "x64"
76  },
77  {
78    "name": "Test ASAN",
79    "os": ubuntu,
80    "filter": "asan",
81    "rust": "wasmtime-ci-pinned-nightly",
82    "target": "x86_64-unknown-linux-gnu",
83  },
84  {
85    "name": "Test Intel SDE",
86    "os": ubuntu,
87    "filter": "sde",
88    "isa": "x64",
89    "sde": true,
90    "crates": "cranelift-tools",
91  },
92  {
93    "name": "Test macOS x86_64",
94    "os": macos,
95    "filter": "macos-x64",
96    "target": "x86_64-apple-darwin",
97  },
98  {
99    "name": "Test macOS arm64",
100    "os": macos,
101    "filter": "macos-arm64",
102    "target": "aarch64-apple-darwin",
103  },
104  {
105    "name": "Test MSVC x86_64",
106    "os": windows,
107    "filter": "windows-x64",
108  },
109  {
110    "name": "Test MinGW x86_64",
111    "os": windows,
112    "target": "x86_64-pc-windows-gnu",
113    "filter": "mingw-x64"
114  },
115  {
116    "name": "Test Linux arm64",
117    "os": ubuntu + '-arm',
118    "target": "aarch64-unknown-linux-gnu",
119    "filter": "linux-arm64",
120    "isa": "aarch64",
121  },
122  {
123    "name": "Test Linux s390x",
124    // "os": 'ubuntu-24.04-s390x',
125    "os": ubuntu,
126    "target": "s390x-unknown-linux-gnu",
127    "filter": "linux-s390x",
128    "isa": "s390x",
129    "gcc_package": "gcc-s390x-linux-gnu",
130    "gcc": "s390x-linux-gnu-gcc",
131    "qemu": "qemu-s390x -L /usr/s390x-linux-gnu",
132    "qemu_target": "s390x-linux-user",
133  },
134  {
135    "name": "Test Linux riscv64",
136    "os": ubuntu,
137    "target": "riscv64gc-unknown-linux-gnu",
138    "gcc_package": "gcc-riscv64-linux-gnu",
139    "gcc": "riscv64-linux-gnu-gcc",
140    "qemu": "qemu-riscv64 -cpu rv64,v=true,vlen=256,vext_spec=v1.0,zfa=true,zfh=true,zba=true,zbb=true,zbc=true,zbs=true,zbkb=true,zcb=true,zicond=true,zvfh=true -L /usr/riscv64-linux-gnu",
141    "qemu_target": "riscv64-linux-user",
142    "filter": "linux-riscv64",
143    "isa": "riscv64",
144  },
145  {
146    "name": "Tests Linux i686",
147    "os": ubuntu,
148    "target": "i686-unknown-linux-gnu",
149    "gcc_package": "gcc-i686-linux-gnu",
150    "gcc": "i686-linux-gnu-gcc",
151  },
152  {
153    "name": "Tests Linux armv7",
154    "os": ubuntu,
155    "target": "armv7-unknown-linux-gnueabihf",
156    "gcc_package": "gcc-arm-linux-gnueabihf",
157    "gcc": "arm-linux-gnueabihf-gcc",
158    "qemu": "qemu-arm -L /usr/arm-linux-gnueabihf -E LD_LIBRARY_PATH=/usr/arm-linux-gnueabihf/lib",
159    "qemu_target": "arm-linux-user",
160  },
161];
162
163/// Get the workspace's full list of member crates.
164async function getWorkspaceMembers() {
165  // Spawn a `cargo metadata` subprocess, accumulate its JSON output from
166  // `stdout`, and wait for it to exit.
167  const child = spawn("cargo", ["metadata"], { encoding: "utf8" });
168  let data = "";
169  child.stdout.on("data", chunk => data += chunk);
170  await new Promise((resolve, reject) => {
171    child.on("close", resolve);
172    child.on("error", reject);
173  });
174
175  // Get the names of the crates in the workspace from the JSON metadata by
176  // building a package-id to name map and then translating the package-ids
177  // listed as workspace members.
178  const metadata = JSON.parse(data);
179  const id_to_name = {};
180  for (const pkg of metadata.packages) {
181    id_to_name[pkg.id] = pkg.name;
182  }
183  return metadata.workspace_members.map(m => id_to_name[m]);
184}
185
186/// For each given target configuration, shard the workspace's crates into
187/// buckets across that config.
188///
189/// This is essentially a `flat_map` where each config that logically tests all
190/// crates in the workspace is mapped to N sharded configs that each test only a
191/// subset of crates in the workspace. Each sharded config's subset of crates to
192/// test are disjoint from all its siblings, and the union of all these siblings'
193/// crates to test is the full workspace members set.
194///
195/// With some poetic license around a `crates_to_test` key that doesn't actually
196/// exist, logically each element of the input `configs` list gets transformed
197/// like this:
198///
199///     { os: "ubuntu-latest", isa: "x64", ..., crates: "all" }
200///
201///     ==>
202///
203///     [
204///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime"] },
205///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime-cli"] },
206///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime-wasi"] },
207///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["cranelift", "cranelift-codegen", ...] },
208///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime-slab", "cranelift-entity", ...] },
209///       { os: "ubuntu-latest", isa: "x64", ..., crates: ["cranelift-environ", "wasmtime-cli-flags", ...] },
210///       ...
211///     ]
212///
213/// Note that `crates: "all"` is implicit in the input and omitted. Similarly,
214/// `crates: [...]` in each output config is actually implemented via adding a
215/// `bucket` key, which contains the CLI flags we must pass to `cargo` to run
216/// tests for just this config's subset of crates.
217async function shard(configs) {
218  const members = await getWorkspaceMembers();
219
220  // Divide the workspace crates into N disjoint subsets. Crates that are
221  // particularly expensive to compile and test form their own singleton subset.
222  const buckets = Array.from({ length: GENERIC_BUCKETS }, _ => new Set());
223  let i = 0;
224  for (const crate of members) {
225    if (SINGLE_CRATE_BUCKETS.indexOf(crate) != -1) continue;
226    buckets[i].add(crate);
227    i = (i + 1) % GENERIC_BUCKETS;
228  }
229  for (crate of SINGLE_CRATE_BUCKETS) {
230    buckets.push(new Set([crate]));
231  }
232
233  // For each config, expand it into N configs, one for each disjoint set we
234  // created above.
235  const sharded = [];
236  for (const config of configs) {
237    // If crates is specified, don't shard, just use the specified crates
238    if (config.crates) {
239      sharded.push(Object.assign(
240        {},
241        config,
242        {
243          bucket: members
244            .map(c => c === config.crates ? `--package ${c}` : `--exclude ${c}`)
245            .join(" ")
246        }
247      ));
248      continue;
249    }
250
251    let nbucket = 1;
252    for (const bucket of buckets) {
253      let bucket_name = `${nbucket}/${buckets.length}`;
254      if (bucket.size === 1)
255        bucket_name = Array.from(bucket)[0];
256
257      sharded.push(Object.assign(
258        {},
259        config,
260        {
261          name: `${config.name} (${bucket_name})`,
262          // We run tests via `cargo test --workspace`, so exclude crates that
263          // aren't in this bucket, rather than naming only the crates that are
264          // in this bucket.
265          bucket: members
266            .map(c => bucket.has(c) ? `--package ${c}` : `--exclude ${c}`)
267            .join(" "),
268        }
269      ));
270      nbucket += 1;
271    }
272  }
273  return sharded;
274}
275
276async function main() {
277  // Our first argument is a file that is a giant json blob which contains at
278  // least all the messages for all of the commits that were a part of this PR.
279  // This is used to test if any commit message includes a string.
280  const commits = fs.readFileSync(process.argv[2]).toString();
281
282  // The second argument is a file that contains the names of all files modified
283  // for a PR, used for file-based filters.
284  const names = fs.readFileSync(process.argv[3]).toString();
285
286  for (let config of FULL_MATRIX) {
287    if (config.rust === undefined) {
288      config.rust = 'default';
289    }
290  }
291
292  // If the optional third argument to this script is `true` then that means all
293  // tests are being run and no filtering should happen.
294  if (process.argv[4] == 'true') {
295    console.log(JSON.stringify(await shard(FULL_MATRIX), undefined, 2));
296    return;
297  }
298
299  // When we aren't running the full CI matrix, filter configs down to just the
300  // relevant bits based on files changed in this commit or if the commit asks
301  // for a certain config to run.
302  const filtered = FULL_MATRIX.filter(config => {
303    // If an ISA-specific test was modified, then include that ISA config.
304    if (config.isa && names.includes(`cranelift/codegen/src/isa/${config.isa}`)) {
305      return true;
306    }
307
308    // If any runtest was modified, include all ISA configs as runtests can
309    // target any backend.
310    if (names.includes(`cranelift/filetests/filetests/runtests`)) {
311      if (config.isa !== undefined)
312        return true;
313    }
314
315    // If the commit explicitly asks for this test config, then include it.
316    if (config.filter && commits.includes(`prtest:${config.filter}`)) {
317      return true;
318    }
319
320    return false;
321  });
322
323  // If at least one test is being run via our filters then run those tests.
324  if (filtered.length > 0) {
325    console.log(JSON.stringify(await shard(filtered), undefined, 2));
326    return;
327  }
328
329  // Otherwise if nothing else is being run, run the fast subset of the matrix.
330  console.log(JSON.stringify(await shard(FAST_MATRIX), undefined, 2));
331}
332
333main()
334