1 //! Helper script to publish the wasmtime and cranelift suites of crates 2 //! 3 //! See documentation in `docs/contributing-release-process.md` for more 4 //! information, but in a nutshell: 5 //! 6 //! * `./publish bump` - bump crate versions in-tree 7 //! * `./publish verify` - verify crates can be published to crates.io 8 //! * `./publish publish` - actually publish crates to crates.io 9 10 use std::collections::HashMap; 11 use std::env; 12 use std::fs; 13 use std::path::{Path, PathBuf}; 14 use std::process::{Command, ExitStatus, Output, Stdio}; 15 use std::thread; 16 use std::time::Duration; 17 18 // note that this list must be topologically sorted by dependencies 19 const CRATES_TO_PUBLISH: &[&str] = &[ 20 // pulley 21 "cranelift-bitset", 22 "wasmtime-math", 23 "pulley-interpreter", 24 // cranelift 25 "cranelift-isle", 26 "cranelift-entity", 27 "cranelift-bforest", 28 "cranelift-codegen-shared", 29 "cranelift-codegen-meta", 30 "cranelift-egraph", 31 "cranelift-control", 32 "cranelift-codegen", 33 "cranelift-reader", 34 "cranelift-serde", 35 "cranelift-module", 36 "cranelift-frontend", 37 "cranelift-native", 38 "cranelift-object", 39 "cranelift-interpreter", 40 "wasmtime-jit-icache-coherence", 41 "cranelift-jit", 42 "cranelift", 43 // wiggle 44 "wiggle-generate", 45 "wiggle-macro", 46 // winch 47 "winch", 48 // wasmtime 49 "wasmtime-asm-macros", 50 "wasmtime-versioned-export-macros", 51 "wasmtime-slab", 52 "wasmtime-component-util", 53 "wasmtime-wit-bindgen", 54 "wasmtime-component-macro", 55 "wasmtime-jit-debug", 56 "wasmtime-fiber", 57 "wasmtime-environ", 58 "wasmtime-wmemcheck", 59 "wasmtime-cranelift", 60 "wasmtime-cache", 61 "winch-codegen", 62 "wasmtime-winch", 63 "wasmtime", 64 // wasi-common/wiggle 65 "wiggle", 66 "wasi-common", 67 // other misc wasmtime crates 68 "wasmtime-wasi", 69 "wasmtime-wasi-http", 70 "wasmtime-wasi-nn", 71 "wasmtime-wasi-config", 72 "wasmtime-wasi-keyvalue", 73 "wasmtime-wasi-threads", 74 "wasmtime-wast", 75 "wasmtime-c-api-macros", 76 "wasmtime-c-api-impl", 77 "wasmtime-cli-flags", 78 "wasmtime-explorer", 79 "wasmtime-cli", 80 ]; 81 82 // Anything **not** mentioned in this array is required to have an `=a.b.c` 83 // dependency requirement on it to enable breaking api changes even in "patch" 84 // releases since everything not mentioned here is just an organizational detail 85 // that no one else should rely on. 86 const PUBLIC_CRATES: &[&str] = &[ 87 // these are actually public crates which we cannot break the API of in 88 // patch releases. 89 "wasmtime", 90 "wasmtime-wasi", 91 "wasmtime-wasi-nn", 92 "wasmtime-wasi-config", 93 "wasmtime-wasi-keyvalue", 94 "wasmtime-wasi-threads", 95 "wasmtime-cli", 96 // all cranelift crates are considered "public" in that they can't 97 // have breaking API changes in patch releases 98 "cranelift-entity", 99 "cranelift-bforest", 100 "cranelift-bitset", 101 "cranelift-codegen-shared", 102 "cranelift-codegen-meta", 103 "cranelift-egraph", 104 "cranelift-control", 105 "cranelift-codegen", 106 "cranelift-reader", 107 "cranelift-serde", 108 "cranelift-module", 109 "cranelift-frontend", 110 "cranelift-native", 111 "cranelift-object", 112 "cranelift-interpreter", 113 "cranelift", 114 "cranelift-jit", 115 // This is a dependency of cranelift crates and as a result can't break in 116 // patch releases as well 117 "wasmtime-types", 118 ]; 119 120 const C_HEADER_PATH: &str = "./crates/c-api/include/wasmtime.h"; 121 122 struct Workspace { 123 version: String, 124 } 125 126 struct Crate { 127 manifest: PathBuf, 128 name: String, 129 version: String, 130 publish: bool, 131 } 132 133 fn main() { 134 let mut crates = Vec::new(); 135 let root = read_crate(None, "./Cargo.toml".as_ref()); 136 let ws = Workspace { 137 version: root.version.clone(), 138 }; 139 crates.push(root); 140 find_crates("crates".as_ref(), &ws, &mut crates); 141 find_crates("cranelift".as_ref(), &ws, &mut crates); 142 find_crates("pulley".as_ref(), &ws, &mut crates); 143 find_crates("winch".as_ref(), &ws, &mut crates); 144 145 let pos = CRATES_TO_PUBLISH 146 .iter() 147 .enumerate() 148 .map(|(i, c)| (*c, i)) 149 .collect::<HashMap<_, _>>(); 150 crates.sort_by_key(|krate| pos.get(&krate.name[..])); 151 152 match &env::args().nth(1).expect("must have one argument")[..] { 153 name @ "bump" | name @ "bump-patch" => { 154 for krate in crates.iter() { 155 bump_version(&krate, &crates, name == "bump-patch"); 156 } 157 // update C API version in wasmtime.h 158 update_capi_version(); 159 // update the lock file 160 run_cmd(Command::new("cargo").arg("fetch")); 161 } 162 163 "publish" => { 164 // We have so many crates to publish we're frequently either 165 // rate-limited or we run into issues where crates can't publish 166 // successfully because they're waiting on the index entries of 167 // previously-published crates to propagate. This means we try to 168 // publish in a loop and we remove crates once they're successfully 169 // published. Failed-to-publish crates get enqueued for another try 170 // later on. 171 for _ in 0..10 { 172 crates.retain(|krate| !publish(krate)); 173 174 if crates.is_empty() { 175 break; 176 } 177 178 println!( 179 "{} crates failed to publish, waiting for a bit to retry", 180 crates.len(), 181 ); 182 thread::sleep(Duration::from_secs(40)); 183 } 184 185 assert!(crates.is_empty(), "failed to publish all crates"); 186 187 println!(""); 188 println!("==================================================================="); 189 println!(""); 190 println!("Don't forget to push a git tag for this release!"); 191 println!(""); 192 println!(" $ git tag vX.Y.Z"); 193 println!(" $ git push [email protected]:bytecodealliance/wasmtime.git vX.Y.Z"); 194 } 195 196 "verify" => { 197 verify(&crates); 198 } 199 200 s => panic!("unknown command: {}", s), 201 } 202 } 203 204 fn cmd_output(cmd: &mut Command) -> Output { 205 eprintln!("Running: `{:?}`", cmd); 206 match cmd.output() { 207 Ok(o) => o, 208 Err(e) => panic!("Failed to run `{:?}`: {}", cmd, e), 209 } 210 } 211 212 fn cmd_status(cmd: &mut Command) -> ExitStatus { 213 eprintln!("Running: `{:?}`", cmd); 214 match cmd.status() { 215 Ok(s) => s, 216 Err(e) => panic!("Failed to run `{:?}`: {}", cmd, e), 217 } 218 } 219 220 fn run_cmd(cmd: &mut Command) { 221 let status = cmd_status(cmd); 222 assert!( 223 status.success(), 224 "Command `{:?}` exited with failure status: {}", 225 cmd, 226 status 227 ); 228 } 229 230 fn find_crates(dir: &Path, ws: &Workspace, dst: &mut Vec<Crate>) { 231 if dir.join("Cargo.toml").exists() { 232 let krate = read_crate(Some(ws), &dir.join("Cargo.toml")); 233 if !krate.publish || CRATES_TO_PUBLISH.iter().any(|c| krate.name == *c) { 234 dst.push(krate); 235 } else { 236 panic!("failed to find {:?} in whitelist or blacklist", krate.name); 237 } 238 } 239 240 for entry in dir.read_dir().unwrap() { 241 let entry = entry.unwrap(); 242 if entry.file_type().unwrap().is_dir() { 243 find_crates(&entry.path(), ws, dst); 244 } 245 } 246 } 247 248 fn read_crate(ws: Option<&Workspace>, manifest: &Path) -> Crate { 249 let mut name = None; 250 let mut version = None; 251 let mut publish = true; 252 for line in fs::read_to_string(manifest).unwrap().lines() { 253 if name.is_none() && line.starts_with("name = \"") { 254 name = Some( 255 line.replace("name = \"", "") 256 .replace("\"", "") 257 .trim() 258 .to_string(), 259 ); 260 } 261 if version.is_none() && line.starts_with("version = \"") { 262 version = Some( 263 line.replace("version = \"", "") 264 .replace("\"", "") 265 .trim() 266 .to_string(), 267 ); 268 } 269 if let Some(ws) = ws { 270 if version.is_none() && line.starts_with("version.workspace = true") { 271 version = Some(ws.version.clone()); 272 } 273 } 274 if line.starts_with("publish = false") { 275 publish = false; 276 } 277 } 278 let name = name.unwrap(); 279 let version = version.unwrap(); 280 Crate { 281 manifest: manifest.to_path_buf(), 282 name, 283 version, 284 publish, 285 } 286 } 287 288 fn bump_version(krate: &Crate, crates: &[Crate], patch: bool) { 289 let contents = fs::read_to_string(&krate.manifest).unwrap(); 290 let next_version = |krate: &Crate| -> String { 291 if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { 292 bump(&krate.version, patch) 293 } else { 294 krate.version.clone() 295 } 296 }; 297 298 let mut new_manifest = String::new(); 299 let mut is_deps = false; 300 for line in contents.lines() { 301 let mut rewritten = false; 302 if !is_deps && line.starts_with("version =") { 303 if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { 304 println!( 305 "bump `{}` {} => {}", 306 krate.name, 307 krate.version, 308 next_version(krate), 309 ); 310 new_manifest.push_str(&line.replace(&krate.version, &next_version(krate))); 311 rewritten = true; 312 } 313 } 314 315 is_deps = if line.starts_with("[") { 316 line.contains("dependencies") 317 } else { 318 is_deps 319 }; 320 321 for other in crates { 322 // If `other` isn't a published crate then it's not going to get a 323 // bumped version so we don't need to update anything in the 324 // manifest. 325 if !other.publish { 326 continue; 327 } 328 if !is_deps || !line.starts_with(&format!("{} ", other.name)) { 329 continue; 330 } 331 if !line.contains(&other.version) { 332 if !line.contains("version =") || !krate.publish { 333 continue; 334 } 335 panic!( 336 "{:?} has a dep on {} but doesn't list version {}", 337 krate.manifest, other.name, other.version 338 ); 339 } 340 if krate.publish { 341 if PUBLIC_CRATES.contains(&other.name.as_str()) { 342 assert!( 343 !line.contains("\"="), 344 "{} should not have an exact version requirement on {}", 345 krate.name, 346 other.name 347 ); 348 } else { 349 assert!( 350 line.contains("\"="), 351 "{} should have an exact version requirement on {}", 352 krate.name, 353 other.name 354 ); 355 } 356 } 357 rewritten = true; 358 new_manifest.push_str(&line.replace(&other.version, &next_version(other))); 359 break; 360 } 361 if !rewritten { 362 new_manifest.push_str(line); 363 } 364 new_manifest.push_str("\n"); 365 } 366 fs::write(&krate.manifest, new_manifest).unwrap(); 367 } 368 369 fn update_capi_version() { 370 let version = read_crate(None, "./Cargo.toml".as_ref()).version; 371 372 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 373 let major = iter.next().expect("major version"); 374 let minor = iter.next().expect("minor version"); 375 let patch = iter.next().expect("patch version"); 376 377 let mut new_header = String::new(); 378 let contents = fs::read_to_string(C_HEADER_PATH).unwrap(); 379 for line in contents.lines() { 380 if line.starts_with("#define WASMTIME_VERSION \"") { 381 new_header.push_str(&format!("#define WASMTIME_VERSION \"{version}\"")); 382 } else if line.starts_with("#define WASMTIME_VERSION_MAJOR") { 383 new_header.push_str(&format!("#define WASMTIME_VERSION_MAJOR {major}")); 384 } else if line.starts_with("#define WASMTIME_VERSION_MINOR") { 385 new_header.push_str(&format!("#define WASMTIME_VERSION_MINOR {minor}")); 386 } else if line.starts_with("#define WASMTIME_VERSION_PATCH") { 387 new_header.push_str(&format!("#define WASMTIME_VERSION_PATCH {patch}")); 388 } else { 389 new_header.push_str(line); 390 } 391 new_header.push_str("\n"); 392 } 393 394 fs::write(&C_HEADER_PATH, new_header).unwrap(); 395 } 396 397 /// Performs a major version bump increment on the semver version `version`. 398 /// 399 /// This function will perform a semver-major-version bump on the `version` 400 /// specified. This is used to calculate the next version of a crate in this 401 /// repository since we're currently making major version bumps for all our 402 /// releases. This may end up getting tweaked as we stabilize crates and start 403 /// doing more minor/patch releases, but for now this should do the trick. 404 fn bump(version: &str, patch_bump: bool) -> String { 405 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 406 let major = iter.next().expect("major version"); 407 let minor = iter.next().expect("minor version"); 408 let patch = iter.next().expect("patch version"); 409 410 if patch_bump { 411 return format!("{}.{}.{}", major, minor, patch + 1); 412 } 413 if major != 0 { 414 format!("{}.0.0", major + 1) 415 } else if minor != 0 { 416 format!("0.{}.0", minor + 1) 417 } else { 418 format!("0.0.{}", patch + 1) 419 } 420 } 421 422 fn publish(krate: &Crate) -> bool { 423 if !CRATES_TO_PUBLISH.iter().any(|s| *s == krate.name) { 424 return true; 425 } 426 427 // First make sure the crate isn't already published at this version. This 428 // script may be re-run and there's no need to re-attempt previous work. 429 let output = cmd_output(Command::new("curl").arg(&format!( 430 "https://crates.io/api/v1/crates/{}/versions", 431 krate.name 432 ))); 433 if output.status.success() 434 && String::from_utf8_lossy(&output.stdout) 435 .contains(&format!("\"num\":\"{}\"", krate.version)) 436 { 437 println!( 438 "skip publish {} because {} is already published", 439 krate.name, krate.version, 440 ); 441 return true; 442 } 443 444 let status = cmd_status( 445 Command::new("cargo") 446 .arg("publish") 447 .current_dir(krate.manifest.parent().unwrap()) 448 .arg("--no-verify"), 449 ); 450 if !status.success() { 451 println!("FAIL: failed to publish `{}`: {}", krate.name, status); 452 return false; 453 } 454 455 // After we've published then make sure that the `wasmtime-publish` group is 456 // added to this crate for future publications. If it's already present 457 // though we can skip the `cargo owner` modification. 458 let output = cmd_output(Command::new("curl").arg(&format!( 459 "https://crates.io/api/v1/crates/{}/owners", 460 krate.name 461 ))); 462 if output.status.success() 463 && String::from_utf8_lossy(&output.stdout).contains("wasmtime-publish") 464 { 465 println!( 466 "wasmtime-publish already listed as an owner of {}", 467 krate.name 468 ); 469 return true; 470 } 471 472 // Note that the status is ignored here. This fails most of the time because 473 // the owner is already set and present, so we only want to add this to 474 // crates which haven't previously been published. 475 run_cmd( 476 Command::new("cargo") 477 .arg("owner") 478 .arg("-a") 479 .arg("github:bytecodealliance:wasmtime-publish") 480 .arg(&krate.name), 481 ); 482 483 true 484 } 485 486 // Verify the current tree is publish-able to crates.io. The intention here is 487 // that we'll run `cargo package` on everything which verifies the build as-if 488 // it were published to crates.io. This requires using an incrementally-built 489 // directory registry generated from `cargo vendor` because the versions 490 // referenced from `Cargo.toml` may not exist on crates.io. 491 fn verify(crates: &[Crate]) { 492 verify_capi(); 493 494 if Path::new(".cargo").exists() { 495 panic!( 496 "`.cargo` already exists on the file system, remove it and then run the script again" 497 ); 498 } 499 if Path::new("vendor").exists() { 500 panic!( 501 "`vendor` already exists on the file system, remove it and then run the script again" 502 ); 503 } 504 505 let vendor = cmd_output(Command::new("cargo").arg("vendor").stderr(Stdio::inherit())); 506 assert!(vendor.status.success()); 507 508 fs::create_dir_all(".cargo").unwrap(); 509 fs::write(".cargo/config.toml", vendor.stdout).unwrap(); 510 511 for krate in crates { 512 if !krate.publish { 513 continue; 514 } 515 verify_and_vendor(&krate); 516 } 517 518 fn verify_and_vendor(krate: &Crate) { 519 let mut cmd = Command::new("cargo"); 520 cmd.arg("package") 521 .arg("--manifest-path") 522 .arg(&krate.manifest) 523 .env("CARGO_TARGET_DIR", "./target"); 524 if krate.name.contains("wasi-nn") { 525 cmd.arg("--no-verify"); 526 } 527 run_cmd(&mut cmd); 528 run_cmd( 529 Command::new("tar") 530 .arg("xf") 531 .arg(format!( 532 "../target/package/{}-{}.crate", 533 krate.name, krate.version 534 )) 535 .current_dir("./vendor"), 536 ); 537 fs::write( 538 format!( 539 "./vendor/{}-{}/.cargo-checksum.json", 540 krate.name, krate.version 541 ), 542 "{\"files\":{}}", 543 ) 544 .unwrap(); 545 } 546 547 fn verify_capi() { 548 let version = read_crate(None, "./Cargo.toml".as_ref()).version; 549 550 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 551 let major = iter.next().expect("major version"); 552 let minor = iter.next().expect("minor version"); 553 let patch = iter.next().expect("patch version"); 554 555 let mut count = 0; 556 let contents = fs::read_to_string(C_HEADER_PATH).unwrap(); 557 for line in contents.lines() { 558 if line.starts_with(&format!("#define WASMTIME_VERSION \"{version}\"")) { 559 count += 1; 560 } else if line.starts_with(&format!("#define WASMTIME_VERSION_MAJOR {major}")) { 561 count += 1; 562 } else if line.starts_with(&format!("#define WASMTIME_VERSION_MINOR {minor}")) { 563 count += 1; 564 } else if line.starts_with(&format!("#define WASMTIME_VERSION_PATCH {patch}")) { 565 count += 1; 566 } 567 } 568 569 assert!( 570 count == 4, 571 "invalid version macros in {}, should match \"{}\"", 572 C_HEADER_PATH, 573 version 574 ); 575 } 576 } 577