1 //! Helper script to publish the wasmtime and cranelift suites of crates 2 //! 3 //! See documentation in `docs/contributing-release-process.md` for more 4 //! information, but in a nutshell: 5 //! 6 //! * `./publish bump` - bump crate versions in-tree 7 //! * `./publish verify` - verify crates can be published to crates.io 8 //! * `./publish publish` - actually publish crates to crates.io 9 10 use std::collections::HashMap; 11 use std::env; 12 use std::fs; 13 use std::path::{Path, PathBuf}; 14 use std::process::{Command, ExitStatus, Output, Stdio}; 15 use std::thread; 16 use std::time::Duration; 17 18 // note that this list must be topologically sorted by dependencies 19 const CRATES_TO_PUBLISH: &[&str] = &[ 20 // pulley 21 "cranelift-bitset", 22 "wasmtime-math", 23 "pulley-interpreter", 24 // cranelift 25 "cranelift-srcgen", 26 "cranelift-assembler-x64-meta", 27 "cranelift-assembler-x64", 28 "cranelift-isle", 29 "cranelift-entity", 30 "cranelift-bforest", 31 "cranelift-codegen-shared", 32 "cranelift-codegen-meta", 33 "cranelift-egraph", 34 "cranelift-control", 35 "cranelift-codegen", 36 "cranelift-reader", 37 "cranelift-serde", 38 "cranelift-module", 39 "cranelift-frontend", 40 "cranelift-native", 41 "cranelift-object", 42 "cranelift-interpreter", 43 "wasmtime-jit-icache-coherence", 44 "cranelift-jit", 45 "cranelift", 46 // wiggle 47 "wiggle-generate", 48 "wiggle-macro", 49 // winch 50 "winch", 51 // wasmtime 52 "wasmtime-asm-macros", 53 "wasmtime-versioned-export-macros", 54 "wasmtime-slab", 55 "wasmtime-component-util", 56 "wasmtime-wit-bindgen", 57 "wasmtime-component-macro", 58 "wasmtime-jit-debug", 59 "wasmtime-fiber", 60 "wasmtime-environ", 61 "wasmtime-wmemcheck", 62 "wasmtime-cranelift", 63 "wasmtime-cache", 64 "winch-codegen", 65 "wasmtime-winch", 66 "wasmtime", 67 // wasi-common/wiggle 68 "wiggle", 69 "wasi-common", 70 // other misc wasmtime crates 71 "wasmtime-wasi-io", 72 "wasmtime-wasi", 73 "wasmtime-wasi-http", 74 "wasmtime-wasi-nn", 75 "wasmtime-wasi-config", 76 "wasmtime-wasi-keyvalue", 77 "wasmtime-wasi-threads", 78 "wasmtime-wasi-tls", 79 "wasmtime-wast", 80 "wasmtime-c-api-macros", 81 "wasmtime-c-api-impl", 82 "wasmtime-cli-flags", 83 "wasmtime-explorer", 84 "wasmtime-cli", 85 ]; 86 87 // Anything **not** mentioned in this array is required to have an `=a.b.c` 88 // dependency requirement on it to enable breaking api changes even in "patch" 89 // releases since everything not mentioned here is just an organizational detail 90 // that no one else should rely on. 91 const PUBLIC_CRATES: &[&str] = &[ 92 // These are actually public crates which we cannot break the API of in 93 // patch releases. 94 "wasmtime", 95 "wasmtime-wasi-io", 96 "wasmtime-wasi", 97 "wasmtime-wasi-tls", 98 "wasmtime-wasi-http", 99 "wasmtime-wasi-nn", 100 "wasmtime-wasi-config", 101 "wasmtime-wasi-keyvalue", 102 "wasmtime-wasi-threads", 103 "wasmtime-cli", 104 // All cranelift crates are considered "public" in that they can't have 105 // breaking API changes in patch releases. 106 "cranelift-srcgen", 107 "cranelift-assembler-x64-meta", 108 "cranelift-assembler-x64", 109 "cranelift-entity", 110 "cranelift-bforest", 111 "cranelift-bitset", 112 "cranelift-codegen-shared", 113 "cranelift-codegen-meta", 114 "cranelift-egraph", 115 "cranelift-control", 116 "cranelift-codegen", 117 "cranelift-reader", 118 "cranelift-serde", 119 "cranelift-module", 120 "cranelift-frontend", 121 "cranelift-native", 122 "cranelift-object", 123 "cranelift-interpreter", 124 "cranelift", 125 "cranelift-jit", 126 // This is a dependency of cranelift crates and as a result can't break in 127 // patch releases as well 128 "wasmtime-types", 129 ]; 130 131 const C_HEADER_PATH: &str = "./crates/c-api/include/wasmtime.h"; 132 133 struct Workspace { 134 version: String, 135 } 136 137 struct Crate { 138 manifest: PathBuf, 139 name: String, 140 version: String, 141 publish: bool, 142 } 143 144 fn main() { 145 let mut crates = Vec::new(); 146 let root = read_crate(None, "./Cargo.toml".as_ref()); 147 let ws = Workspace { 148 version: root.version.clone(), 149 }; 150 crates.push(root); 151 find_crates("crates".as_ref(), &ws, &mut crates); 152 find_crates("cranelift".as_ref(), &ws, &mut crates); 153 find_crates("pulley".as_ref(), &ws, &mut crates); 154 find_crates("winch".as_ref(), &ws, &mut crates); 155 156 let pos = CRATES_TO_PUBLISH 157 .iter() 158 .enumerate() 159 .map(|(i, c)| (*c, i)) 160 .collect::<HashMap<_, _>>(); 161 crates.sort_by_key(|krate| pos.get(&krate.name[..])); 162 163 match &env::args().nth(1).expect("must have one argument")[..] { 164 name @ "bump" | name @ "bump-patch" => { 165 for krate in crates.iter() { 166 bump_version(&krate, &crates, name == "bump-patch"); 167 } 168 // update C API version in wasmtime.h 169 update_capi_version(); 170 // update the lock file 171 run_cmd(Command::new("cargo").arg("fetch")); 172 } 173 174 "publish" => { 175 // We have so many crates to publish we're frequently either 176 // rate-limited or we run into issues where crates can't publish 177 // successfully because they're waiting on the index entries of 178 // previously-published crates to propagate. This means we try to 179 // publish in a loop and we remove crates once they're successfully 180 // published. Failed-to-publish crates get enqueued for another try 181 // later on. 182 for _ in 0..10 { 183 crates.retain(|krate| !publish(krate)); 184 185 if crates.is_empty() { 186 break; 187 } 188 189 println!( 190 "{} crates failed to publish, waiting for a bit to retry", 191 crates.len(), 192 ); 193 thread::sleep(Duration::from_secs(40)); 194 } 195 196 assert!(crates.is_empty(), "failed to publish all crates"); 197 198 println!(""); 199 println!("==================================================================="); 200 println!(""); 201 println!("Don't forget to push a git tag for this release!"); 202 println!(""); 203 println!(" $ git tag vX.Y.Z"); 204 println!(" $ git push [email protected]:bytecodealliance/wasmtime.git vX.Y.Z"); 205 } 206 207 "verify" => { 208 verify(&crates); 209 } 210 211 s => panic!("unknown command: {}", s), 212 } 213 } 214 215 fn cmd_output(cmd: &mut Command) -> Output { 216 eprintln!("Running: `{:?}`", cmd); 217 match cmd.output() { 218 Ok(o) => o, 219 Err(e) => panic!("Failed to run `{:?}`: {}", cmd, e), 220 } 221 } 222 223 fn cmd_status(cmd: &mut Command) -> ExitStatus { 224 eprintln!("Running: `{:?}`", cmd); 225 match cmd.status() { 226 Ok(s) => s, 227 Err(e) => panic!("Failed to run `{:?}`: {}", cmd, e), 228 } 229 } 230 231 fn run_cmd(cmd: &mut Command) { 232 let status = cmd_status(cmd); 233 assert!( 234 status.success(), 235 "Command `{:?}` exited with failure status: {}", 236 cmd, 237 status 238 ); 239 } 240 241 fn find_crates(dir: &Path, ws: &Workspace, dst: &mut Vec<Crate>) { 242 if dir.join("Cargo.toml").exists() { 243 let krate = read_crate(Some(ws), &dir.join("Cargo.toml")); 244 if !krate.publish || CRATES_TO_PUBLISH.iter().any(|c| krate.name == *c) { 245 dst.push(krate); 246 } else { 247 panic!("failed to find {:?} in whitelist or blacklist", krate.name); 248 } 249 } 250 251 for entry in dir.read_dir().unwrap() { 252 let entry = entry.unwrap(); 253 if entry.file_type().unwrap().is_dir() { 254 find_crates(&entry.path(), ws, dst); 255 } 256 } 257 } 258 259 fn read_crate(ws: Option<&Workspace>, manifest: &Path) -> Crate { 260 let mut name = None; 261 let mut version = None; 262 let mut publish = true; 263 for line in fs::read_to_string(manifest).unwrap().lines() { 264 if name.is_none() && line.starts_with("name = \"") { 265 name = Some( 266 line.replace("name = \"", "") 267 .replace("\"", "") 268 .trim() 269 .to_string(), 270 ); 271 } 272 if version.is_none() && line.starts_with("version = \"") { 273 version = Some( 274 line.replace("version = \"", "") 275 .replace("\"", "") 276 .trim() 277 .to_string(), 278 ); 279 } 280 if let Some(ws) = ws { 281 if version.is_none() && line.starts_with("version.workspace = true") { 282 version = Some(ws.version.clone()); 283 } 284 } 285 if line.starts_with("publish = false") { 286 publish = false; 287 } 288 } 289 let name = name.unwrap(); 290 let version = version.unwrap(); 291 Crate { 292 manifest: manifest.to_path_buf(), 293 name, 294 version, 295 publish, 296 } 297 } 298 299 fn bump_version(krate: &Crate, crates: &[Crate], patch: bool) { 300 let contents = fs::read_to_string(&krate.manifest).unwrap(); 301 let next_version = |krate: &Crate| -> String { 302 if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { 303 bump(&krate.version, patch) 304 } else { 305 krate.version.clone() 306 } 307 }; 308 309 let mut new_manifest = String::new(); 310 let mut is_deps = false; 311 for line in contents.lines() { 312 let mut rewritten = false; 313 if !is_deps && line.starts_with("version =") { 314 if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { 315 println!( 316 "bump `{}` {} => {}", 317 krate.name, 318 krate.version, 319 next_version(krate), 320 ); 321 new_manifest.push_str(&line.replace(&krate.version, &next_version(krate))); 322 rewritten = true; 323 } 324 } 325 326 is_deps = if line.starts_with("[") { 327 line.contains("dependencies") 328 } else { 329 is_deps 330 }; 331 332 for other in crates { 333 // If `other` isn't a published crate then it's not going to get a 334 // bumped version so we don't need to update anything in the 335 // manifest. 336 if !other.publish { 337 continue; 338 } 339 if !is_deps || !line.starts_with(&format!("{} ", other.name)) { 340 continue; 341 } 342 if !line.contains(&other.version) { 343 if !line.contains("version =") || !krate.publish { 344 continue; 345 } 346 panic!( 347 "{:?} has a dep on {} but doesn't list version {}", 348 krate.manifest, other.name, other.version 349 ); 350 } 351 if krate.publish { 352 if PUBLIC_CRATES.contains(&other.name.as_str()) { 353 assert!( 354 !line.contains("\"="), 355 "{} should not have an exact version requirement on {}", 356 krate.name, 357 other.name 358 ); 359 } else { 360 assert!( 361 line.contains("\"="), 362 "{} should have an exact version requirement on {}", 363 krate.name, 364 other.name 365 ); 366 } 367 } 368 rewritten = true; 369 new_manifest.push_str(&line.replace(&other.version, &next_version(other))); 370 break; 371 } 372 if !rewritten { 373 new_manifest.push_str(line); 374 } 375 new_manifest.push_str("\n"); 376 } 377 fs::write(&krate.manifest, new_manifest).unwrap(); 378 } 379 380 fn update_capi_version() { 381 let version = read_crate(None, "./Cargo.toml".as_ref()).version; 382 383 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 384 let major = iter.next().expect("major version"); 385 let minor = iter.next().expect("minor version"); 386 let patch = iter.next().expect("patch version"); 387 388 let mut new_header = String::new(); 389 let contents = fs::read_to_string(C_HEADER_PATH).unwrap(); 390 for line in contents.lines() { 391 if line.starts_with("#define WASMTIME_VERSION \"") { 392 new_header.push_str(&format!("#define WASMTIME_VERSION \"{version}\"")); 393 } else if line.starts_with("#define WASMTIME_VERSION_MAJOR") { 394 new_header.push_str(&format!("#define WASMTIME_VERSION_MAJOR {major}")); 395 } else if line.starts_with("#define WASMTIME_VERSION_MINOR") { 396 new_header.push_str(&format!("#define WASMTIME_VERSION_MINOR {minor}")); 397 } else if line.starts_with("#define WASMTIME_VERSION_PATCH") { 398 new_header.push_str(&format!("#define WASMTIME_VERSION_PATCH {patch}")); 399 } else { 400 new_header.push_str(line); 401 } 402 new_header.push_str("\n"); 403 } 404 405 fs::write(&C_HEADER_PATH, new_header).unwrap(); 406 } 407 408 /// Performs a major version bump increment on the semver version `version`. 409 /// 410 /// This function will perform a semver-major-version bump on the `version` 411 /// specified. This is used to calculate the next version of a crate in this 412 /// repository since we're currently making major version bumps for all our 413 /// releases. This may end up getting tweaked as we stabilize crates and start 414 /// doing more minor/patch releases, but for now this should do the trick. 415 fn bump(version: &str, patch_bump: bool) -> String { 416 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 417 let major = iter.next().expect("major version"); 418 let minor = iter.next().expect("minor version"); 419 let patch = iter.next().expect("patch version"); 420 421 if patch_bump { 422 return format!("{}.{}.{}", major, minor, patch + 1); 423 } 424 if major != 0 { 425 format!("{}.0.0", major + 1) 426 } else if minor != 0 { 427 format!("0.{}.0", minor + 1) 428 } else { 429 format!("0.0.{}", patch + 1) 430 } 431 } 432 433 fn publish(krate: &Crate) -> bool { 434 if !CRATES_TO_PUBLISH.iter().any(|s| *s == krate.name) { 435 return true; 436 } 437 438 // First make sure the crate isn't already published at this version. This 439 // script may be re-run and there's no need to re-attempt previous work. 440 let output = cmd_output(Command::new("curl").arg(&format!( 441 "https://crates.io/api/v1/crates/{}/versions", 442 krate.name 443 ))); 444 if output.status.success() 445 && String::from_utf8_lossy(&output.stdout) 446 .contains(&format!("\"num\":\"{}\"", krate.version)) 447 { 448 println!( 449 "skip publish {} because {} is already published", 450 krate.name, krate.version, 451 ); 452 return true; 453 } 454 455 let status = cmd_status( 456 Command::new("cargo") 457 .arg("publish") 458 .current_dir(krate.manifest.parent().unwrap()) 459 .arg("--no-verify"), 460 ); 461 if !status.success() { 462 println!("FAIL: failed to publish `{}`: {}", krate.name, status); 463 return false; 464 } 465 466 // After we've published then make sure that the `wasmtime-publish` group is 467 // added to this crate for future publications. If it's already present 468 // though we can skip the `cargo owner` modification. 469 let output = cmd_output(Command::new("curl").arg(&format!( 470 "https://crates.io/api/v1/crates/{}/owners", 471 krate.name 472 ))); 473 if output.status.success() 474 && String::from_utf8_lossy(&output.stdout).contains("wasmtime-publish") 475 { 476 println!( 477 "wasmtime-publish already listed as an owner of {}", 478 krate.name 479 ); 480 return true; 481 } 482 483 // Note that the status is ignored here. This fails most of the time because 484 // the owner is already set and present, so we only want to add this to 485 // crates which haven't previously been published. 486 run_cmd( 487 Command::new("cargo") 488 .arg("owner") 489 .arg("-a") 490 .arg("github:bytecodealliance:wasmtime-publish") 491 .arg(&krate.name), 492 ); 493 494 true 495 } 496 497 // Verify the current tree is publish-able to crates.io. The intention here is 498 // that we'll run `cargo package` on everything which verifies the build as-if 499 // it were published to crates.io. This requires using an incrementally-built 500 // directory registry generated from `cargo vendor` because the versions 501 // referenced from `Cargo.toml` may not exist on crates.io. 502 fn verify(crates: &[Crate]) { 503 verify_capi(); 504 505 if Path::new(".cargo").exists() { 506 panic!( 507 "`.cargo` already exists on the file system, remove it and then run the script again" 508 ); 509 } 510 if Path::new("vendor").exists() { 511 panic!( 512 "`vendor` already exists on the file system, remove it and then run the script again" 513 ); 514 } 515 516 let vendor = cmd_output(Command::new("cargo").arg("vendor").stderr(Stdio::inherit())); 517 assert!(vendor.status.success()); 518 519 fs::create_dir_all(".cargo").unwrap(); 520 fs::write(".cargo/config.toml", vendor.stdout).unwrap(); 521 522 for krate in crates { 523 if !krate.publish { 524 continue; 525 } 526 verify_and_vendor(&krate); 527 } 528 529 fn verify_and_vendor(krate: &Crate) { 530 let mut cmd = Command::new("cargo"); 531 cmd.arg("package") 532 .arg("--manifest-path") 533 .arg(&krate.manifest) 534 .env("CARGO_TARGET_DIR", "./target"); 535 if krate.name.contains("wasi-nn") { 536 cmd.arg("--no-verify"); 537 } 538 run_cmd(&mut cmd); 539 run_cmd( 540 Command::new("tar") 541 .arg("xf") 542 .arg(format!( 543 "../target/package/{}-{}.crate", 544 krate.name, krate.version 545 )) 546 .current_dir("./vendor"), 547 ); 548 fs::write( 549 format!( 550 "./vendor/{}-{}/.cargo-checksum.json", 551 krate.name, krate.version 552 ), 553 "{\"files\":{}}", 554 ) 555 .unwrap(); 556 } 557 558 fn verify_capi() { 559 let version = read_crate(None, "./Cargo.toml".as_ref()).version; 560 561 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 562 let major = iter.next().expect("major version"); 563 let minor = iter.next().expect("minor version"); 564 let patch = iter.next().expect("patch version"); 565 566 let mut count = 0; 567 let contents = fs::read_to_string(C_HEADER_PATH).unwrap(); 568 for line in contents.lines() { 569 if line.starts_with(&format!("#define WASMTIME_VERSION \"{version}\"")) { 570 count += 1; 571 } else if line.starts_with(&format!("#define WASMTIME_VERSION_MAJOR {major}")) { 572 count += 1; 573 } else if line.starts_with(&format!("#define WASMTIME_VERSION_MINOR {minor}")) { 574 count += 1; 575 } else if line.starts_with(&format!("#define WASMTIME_VERSION_PATCH {patch}")) { 576 count += 1; 577 } 578 } 579 580 assert!( 581 count == 4, 582 "invalid version macros in {}, should match \"{}\"", 583 C_HEADER_PATH, 584 version 585 ); 586 } 587 } 588