1 //! Helper script to publish the wasmtime and cranelift suites of crates 2 //! 3 //! See documentation in `docs/contributing-release-process.md` for more 4 //! information, but in a nutshell: 5 //! 6 //! * `./publish bump` - bump crate versions in-tree 7 //! * `./publish verify` - verify crates can be published to crates.io 8 //! * `./publish publish` - actually publish crates to crates.io 9 10 use std::collections::HashMap; 11 use std::env; 12 use std::fs; 13 use std::path::{Path, PathBuf}; 14 use std::process::{Command, ExitStatus, Output, Stdio}; 15 use std::thread; 16 use std::time::Duration; 17 18 // note that this list must be topologically sorted by dependencies 19 const CRATES_TO_PUBLISH: &[&str] = &[ 20 // pulley 21 "cranelift-bitset", 22 "wasmtime-math", 23 "pulley-macros", 24 "pulley-interpreter", 25 // cranelift 26 "cranelift-srcgen", 27 "cranelift-assembler-x64-meta", 28 "cranelift-assembler-x64", 29 "cranelift-isle", 30 "cranelift-entity", 31 "cranelift-bforest", 32 "cranelift-codegen-shared", 33 "cranelift-codegen-meta", 34 "cranelift-egraph", 35 "cranelift-control", 36 "cranelift-codegen", 37 "cranelift-reader", 38 "cranelift-serde", 39 "cranelift-module", 40 "cranelift-frontend", 41 "cranelift-native", 42 "cranelift-object", 43 "cranelift-interpreter", 44 "wasmtime-jit-icache-coherence", 45 "cranelift-jit", 46 "cranelift", 47 // wiggle 48 "wiggle-generate", 49 "wiggle-macro", 50 // winch 51 "winch", 52 // wasmtime 53 "wasmtime-asm-macros", 54 "wasmtime-versioned-export-macros", 55 "wasmtime-slab", 56 "wasmtime-component-util", 57 "wasmtime-wit-bindgen", 58 "wasmtime-component-macro", 59 "wasmtime-jit-debug", 60 "wasmtime-fiber", 61 "wasmtime-environ", 62 "wasmtime-wmemcheck", 63 "wasmtime-cranelift", 64 "wasmtime-cache", 65 "winch-codegen", 66 "wasmtime-winch", 67 "wasmtime", 68 // wasi-common/wiggle 69 "wiggle", 70 "wasi-common", 71 // other misc wasmtime crates 72 "wasmtime-wasi-io", 73 "wasmtime-wasi", 74 "wasmtime-wasi-http", 75 "wasmtime-wasi-nn", 76 "wasmtime-wasi-config", 77 "wasmtime-wasi-keyvalue", 78 "wasmtime-wasi-threads", 79 "wasmtime-wasi-tls", 80 "wasmtime-wast", 81 "wasmtime-c-api-macros", 82 "wasmtime-c-api-impl", 83 "wasmtime-cli-flags", 84 "wasmtime-explorer", 85 "wasmtime-cli", 86 ]; 87 88 // Anything **not** mentioned in this array is required to have an `=a.b.c` 89 // dependency requirement on it to enable breaking api changes even in "patch" 90 // releases since everything not mentioned here is just an organizational detail 91 // that no one else should rely on. 92 const PUBLIC_CRATES: &[&str] = &[ 93 // These are actually public crates which we cannot break the API of in 94 // patch releases. 95 "wasmtime", 96 "wasmtime-wasi-io", 97 "wasmtime-wasi", 98 "wasmtime-wasi-tls", 99 "wasmtime-wasi-http", 100 "wasmtime-wasi-nn", 101 "wasmtime-wasi-config", 102 "wasmtime-wasi-keyvalue", 103 "wasmtime-wasi-threads", 104 "wasmtime-cli", 105 // All cranelift crates are considered "public" in that they can't have 106 // breaking API changes in patch releases. 107 "cranelift-srcgen", 108 "cranelift-assembler-x64-meta", 109 "cranelift-assembler-x64", 110 "cranelift-entity", 111 "cranelift-bforest", 112 "cranelift-bitset", 113 "cranelift-codegen-shared", 114 "cranelift-codegen-meta", 115 "cranelift-egraph", 116 "cranelift-control", 117 "cranelift-codegen", 118 "cranelift-reader", 119 "cranelift-serde", 120 "cranelift-module", 121 "cranelift-frontend", 122 "cranelift-native", 123 "cranelift-object", 124 "cranelift-interpreter", 125 "cranelift", 126 "cranelift-jit", 127 // This is a dependency of cranelift crates and as a result can't break in 128 // patch releases as well 129 "wasmtime-types", 130 ]; 131 132 const C_HEADER_PATH: &str = "./crates/c-api/include/wasmtime.h"; 133 134 struct Workspace { 135 version: String, 136 } 137 138 struct Crate { 139 manifest: PathBuf, 140 name: String, 141 version: String, 142 publish: bool, 143 } 144 145 fn main() { 146 let mut crates = Vec::new(); 147 let root = read_crate(None, "./Cargo.toml".as_ref()); 148 let ws = Workspace { 149 version: root.version.clone(), 150 }; 151 crates.push(root); 152 find_crates("crates".as_ref(), &ws, &mut crates); 153 find_crates("cranelift".as_ref(), &ws, &mut crates); 154 find_crates("pulley".as_ref(), &ws, &mut crates); 155 find_crates("winch".as_ref(), &ws, &mut crates); 156 157 let pos = CRATES_TO_PUBLISH 158 .iter() 159 .enumerate() 160 .map(|(i, c)| (*c, i)) 161 .collect::<HashMap<_, _>>(); 162 crates.sort_by_key(|krate| pos.get(&krate.name[..])); 163 164 match &env::args().nth(1).expect("must have one argument")[..] { 165 name @ "bump" | name @ "bump-patch" => { 166 for krate in crates.iter() { 167 bump_version(&krate, &crates, name == "bump-patch"); 168 } 169 // update C API version in wasmtime.h 170 update_capi_version(); 171 // update the lock file 172 run_cmd(Command::new("cargo").arg("fetch")); 173 } 174 175 "publish" => { 176 // We have so many crates to publish we're frequently either 177 // rate-limited or we run into issues where crates can't publish 178 // successfully because they're waiting on the index entries of 179 // previously-published crates to propagate. This means we try to 180 // publish in a loop and we remove crates once they're successfully 181 // published. Failed-to-publish crates get enqueued for another try 182 // later on. 183 for _ in 0..10 { 184 crates.retain(|krate| !publish(krate)); 185 186 if crates.is_empty() { 187 break; 188 } 189 190 println!( 191 "{} crates failed to publish, waiting for a bit to retry", 192 crates.len(), 193 ); 194 thread::sleep(Duration::from_secs(40)); 195 } 196 197 assert!(crates.is_empty(), "failed to publish all crates"); 198 199 println!(""); 200 println!("==================================================================="); 201 println!(""); 202 println!("Don't forget to push a git tag for this release!"); 203 println!(""); 204 println!(" $ git tag vX.Y.Z"); 205 println!(" $ git push [email protected]:bytecodealliance/wasmtime.git vX.Y.Z"); 206 } 207 208 "verify" => { 209 verify(&crates); 210 } 211 212 s => panic!("unknown command: {}", s), 213 } 214 } 215 216 fn cmd_output(cmd: &mut Command) -> Output { 217 eprintln!("Running: `{:?}`", cmd); 218 match cmd.output() { 219 Ok(o) => o, 220 Err(e) => panic!("Failed to run `{:?}`: {}", cmd, e), 221 } 222 } 223 224 fn cmd_status(cmd: &mut Command) -> ExitStatus { 225 eprintln!("Running: `{:?}`", cmd); 226 match cmd.status() { 227 Ok(s) => s, 228 Err(e) => panic!("Failed to run `{:?}`: {}", cmd, e), 229 } 230 } 231 232 fn run_cmd(cmd: &mut Command) { 233 let status = cmd_status(cmd); 234 assert!( 235 status.success(), 236 "Command `{:?}` exited with failure status: {}", 237 cmd, 238 status 239 ); 240 } 241 242 fn find_crates(dir: &Path, ws: &Workspace, dst: &mut Vec<Crate>) { 243 if dir.join("Cargo.toml").exists() { 244 let krate = read_crate(Some(ws), &dir.join("Cargo.toml")); 245 if !krate.publish || CRATES_TO_PUBLISH.iter().any(|c| krate.name == *c) { 246 dst.push(krate); 247 } else { 248 panic!("failed to find {:?} in whitelist or blacklist", krate.name); 249 } 250 } 251 252 for entry in dir.read_dir().unwrap() { 253 let entry = entry.unwrap(); 254 if entry.file_type().unwrap().is_dir() { 255 find_crates(&entry.path(), ws, dst); 256 } 257 } 258 } 259 260 fn read_crate(ws: Option<&Workspace>, manifest: &Path) -> Crate { 261 let mut name = None; 262 let mut version = None; 263 let mut publish = true; 264 for line in fs::read_to_string(manifest).unwrap().lines() { 265 if name.is_none() && line.starts_with("name = \"") { 266 name = Some( 267 line.replace("name = \"", "") 268 .replace("\"", "") 269 .trim() 270 .to_string(), 271 ); 272 } 273 if version.is_none() && line.starts_with("version = \"") { 274 version = Some( 275 line.replace("version = \"", "") 276 .replace("\"", "") 277 .trim() 278 .to_string(), 279 ); 280 } 281 if let Some(ws) = ws { 282 if version.is_none() && line.starts_with("version.workspace = true") { 283 version = Some(ws.version.clone()); 284 } 285 } 286 if line.starts_with("publish = false") { 287 publish = false; 288 } 289 } 290 let name = name.unwrap(); 291 let version = version.unwrap(); 292 Crate { 293 manifest: manifest.to_path_buf(), 294 name, 295 version, 296 publish, 297 } 298 } 299 300 fn bump_version(krate: &Crate, crates: &[Crate], patch: bool) { 301 let contents = fs::read_to_string(&krate.manifest).unwrap(); 302 let next_version = |krate: &Crate| -> String { 303 if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { 304 bump(&krate.version, patch) 305 } else { 306 krate.version.clone() 307 } 308 }; 309 310 let mut new_manifest = String::new(); 311 let mut is_deps = false; 312 for line in contents.lines() { 313 let mut rewritten = false; 314 if !is_deps && line.starts_with("version =") { 315 if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { 316 println!( 317 "bump `{}` {} => {}", 318 krate.name, 319 krate.version, 320 next_version(krate), 321 ); 322 new_manifest.push_str(&line.replace(&krate.version, &next_version(krate))); 323 rewritten = true; 324 } 325 } 326 327 is_deps = if line.starts_with("[") { 328 line.contains("dependencies") 329 } else { 330 is_deps 331 }; 332 333 for other in crates { 334 // If `other` isn't a published crate then it's not going to get a 335 // bumped version so we don't need to update anything in the 336 // manifest. 337 if !other.publish { 338 continue; 339 } 340 if !is_deps || !line.starts_with(&format!("{} ", other.name)) { 341 continue; 342 } 343 if !line.contains(&other.version) { 344 if !line.contains("version =") || !krate.publish { 345 continue; 346 } 347 panic!( 348 "{:?} has a dep on {} but doesn't list version {}", 349 krate.manifest, other.name, other.version 350 ); 351 } 352 if krate.publish { 353 if PUBLIC_CRATES.contains(&other.name.as_str()) { 354 assert!( 355 !line.contains("\"="), 356 "{} should not have an exact version requirement on {}", 357 krate.name, 358 other.name 359 ); 360 } else { 361 assert!( 362 line.contains("\"="), 363 "{} should have an exact version requirement on {}", 364 krate.name, 365 other.name 366 ); 367 } 368 } 369 rewritten = true; 370 new_manifest.push_str(&line.replace(&other.version, &next_version(other))); 371 break; 372 } 373 if !rewritten { 374 new_manifest.push_str(line); 375 } 376 new_manifest.push_str("\n"); 377 } 378 fs::write(&krate.manifest, new_manifest).unwrap(); 379 } 380 381 fn update_capi_version() { 382 let version = read_crate(None, "./Cargo.toml".as_ref()).version; 383 384 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 385 let major = iter.next().expect("major version"); 386 let minor = iter.next().expect("minor version"); 387 let patch = iter.next().expect("patch version"); 388 389 let mut new_header = String::new(); 390 let contents = fs::read_to_string(C_HEADER_PATH).unwrap(); 391 for line in contents.lines() { 392 if line.starts_with("#define WASMTIME_VERSION \"") { 393 new_header.push_str(&format!("#define WASMTIME_VERSION \"{version}\"")); 394 } else if line.starts_with("#define WASMTIME_VERSION_MAJOR") { 395 new_header.push_str(&format!("#define WASMTIME_VERSION_MAJOR {major}")); 396 } else if line.starts_with("#define WASMTIME_VERSION_MINOR") { 397 new_header.push_str(&format!("#define WASMTIME_VERSION_MINOR {minor}")); 398 } else if line.starts_with("#define WASMTIME_VERSION_PATCH") { 399 new_header.push_str(&format!("#define WASMTIME_VERSION_PATCH {patch}")); 400 } else { 401 new_header.push_str(line); 402 } 403 new_header.push_str("\n"); 404 } 405 406 fs::write(&C_HEADER_PATH, new_header).unwrap(); 407 } 408 409 /// Performs a major version bump increment on the semver version `version`. 410 /// 411 /// This function will perform a semver-major-version bump on the `version` 412 /// specified. This is used to calculate the next version of a crate in this 413 /// repository since we're currently making major version bumps for all our 414 /// releases. This may end up getting tweaked as we stabilize crates and start 415 /// doing more minor/patch releases, but for now this should do the trick. 416 fn bump(version: &str, patch_bump: bool) -> String { 417 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 418 let major = iter.next().expect("major version"); 419 let minor = iter.next().expect("minor version"); 420 let patch = iter.next().expect("patch version"); 421 422 if patch_bump { 423 return format!("{}.{}.{}", major, minor, patch + 1); 424 } 425 if major != 0 { 426 format!("{}.0.0", major + 1) 427 } else if minor != 0 { 428 format!("0.{}.0", minor + 1) 429 } else { 430 format!("0.0.{}", patch + 1) 431 } 432 } 433 434 fn publish(krate: &Crate) -> bool { 435 if !CRATES_TO_PUBLISH.iter().any(|s| *s == krate.name) { 436 return true; 437 } 438 439 // First make sure the crate isn't already published at this version. This 440 // script may be re-run and there's no need to re-attempt previous work. 441 let output = cmd_output(Command::new("curl").arg(&format!( 442 "https://crates.io/api/v1/crates/{}/versions", 443 krate.name 444 ))); 445 if output.status.success() 446 && String::from_utf8_lossy(&output.stdout) 447 .contains(&format!("\"num\":\"{}\"", krate.version)) 448 { 449 println!( 450 "skip publish {} because {} is already published", 451 krate.name, krate.version, 452 ); 453 return true; 454 } 455 456 let status = cmd_status( 457 Command::new("cargo") 458 .arg("publish") 459 .current_dir(krate.manifest.parent().unwrap()) 460 .arg("--no-verify"), 461 ); 462 if !status.success() { 463 println!("FAIL: failed to publish `{}`: {}", krate.name, status); 464 return false; 465 } 466 467 // After we've published then make sure that the `wasmtime-publish` group is 468 // added to this crate for future publications. If it's already present 469 // though we can skip the `cargo owner` modification. 470 let output = cmd_output(Command::new("curl").arg(&format!( 471 "https://crates.io/api/v1/crates/{}/owners", 472 krate.name 473 ))); 474 if output.status.success() 475 && String::from_utf8_lossy(&output.stdout).contains("wasmtime-publish") 476 { 477 println!( 478 "wasmtime-publish already listed as an owner of {}", 479 krate.name 480 ); 481 return true; 482 } 483 484 // Note that the status is ignored here. This fails most of the time because 485 // the owner is already set and present, so we only want to add this to 486 // crates which haven't previously been published. 487 run_cmd( 488 Command::new("cargo") 489 .arg("owner") 490 .arg("-a") 491 .arg("github:bytecodealliance:wasmtime-publish") 492 .arg(&krate.name), 493 ); 494 495 true 496 } 497 498 // Verify the current tree is publish-able to crates.io. The intention here is 499 // that we'll run `cargo package` on everything which verifies the build as-if 500 // it were published to crates.io. This requires using an incrementally-built 501 // directory registry generated from `cargo vendor` because the versions 502 // referenced from `Cargo.toml` may not exist on crates.io. 503 fn verify(crates: &[Crate]) { 504 verify_capi(); 505 506 if Path::new(".cargo").exists() { 507 panic!( 508 "`.cargo` already exists on the file system, remove it and then run the script again" 509 ); 510 } 511 if Path::new("vendor").exists() { 512 panic!( 513 "`vendor` already exists on the file system, remove it and then run the script again" 514 ); 515 } 516 517 let vendor = cmd_output(Command::new("cargo").arg("vendor").stderr(Stdio::inherit())); 518 assert!(vendor.status.success()); 519 520 fs::create_dir_all(".cargo").unwrap(); 521 fs::write(".cargo/config.toml", vendor.stdout).unwrap(); 522 523 for krate in crates { 524 if !krate.publish { 525 continue; 526 } 527 verify_and_vendor(&krate); 528 } 529 530 fn verify_and_vendor(krate: &Crate) { 531 let mut cmd = Command::new("cargo"); 532 cmd.arg("package") 533 .arg("--manifest-path") 534 .arg(&krate.manifest) 535 .env("CARGO_TARGET_DIR", "./target"); 536 if krate.name.contains("wasi-nn") { 537 cmd.arg("--no-verify"); 538 } 539 run_cmd(&mut cmd); 540 run_cmd( 541 Command::new("tar") 542 .arg("xf") 543 .arg(format!( 544 "../target/package/{}-{}.crate", 545 krate.name, krate.version 546 )) 547 .current_dir("./vendor"), 548 ); 549 fs::write( 550 format!( 551 "./vendor/{}-{}/.cargo-checksum.json", 552 krate.name, krate.version 553 ), 554 "{\"files\":{}}", 555 ) 556 .unwrap(); 557 } 558 559 fn verify_capi() { 560 let version = read_crate(None, "./Cargo.toml".as_ref()).version; 561 562 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 563 let major = iter.next().expect("major version"); 564 let minor = iter.next().expect("minor version"); 565 let patch = iter.next().expect("patch version"); 566 567 let mut count = 0; 568 let contents = fs::read_to_string(C_HEADER_PATH).unwrap(); 569 for line in contents.lines() { 570 if line.starts_with(&format!("#define WASMTIME_VERSION \"{version}\"")) { 571 count += 1; 572 } else if line.starts_with(&format!("#define WASMTIME_VERSION_MAJOR {major}")) { 573 count += 1; 574 } else if line.starts_with(&format!("#define WASMTIME_VERSION_MINOR {minor}")) { 575 count += 1; 576 } else if line.starts_with(&format!("#define WASMTIME_VERSION_PATCH {patch}")) { 577 count += 1; 578 } 579 } 580 581 assert!( 582 count == 4, 583 "invalid version macros in {}, should match \"{}\"", 584 C_HEADER_PATH, 585 version 586 ); 587 } 588 } 589