1 //! Helper script to publish the wasmtime and cranelift suites of crates 2 //! 3 //! See documentation in `docs/contributing-release-process.md` for more 4 //! information, but in a nutshell: 5 //! 6 //! * `./publish bump` - bump crate versions in-tree 7 //! * `./publish verify` - verify crates can be published to crates.io 8 //! * `./publish publish` - actually publish crates to crates.io 9 10 use std::collections::HashMap; 11 use std::env; 12 use std::fs; 13 use std::path::{Path, PathBuf}; 14 use std::process::{Command, ExitStatus, Output, Stdio}; 15 use std::thread; 16 use std::time::Duration; 17 18 // note that this list must be topologically sorted by dependencies 19 const CRATES_TO_PUBLISH: &[&str] = &[ 20 // pulley 21 "cranelift-bitset", 22 "wasmtime-math", 23 "pulley-interpreter", 24 // cranelift 25 "cranelift-isle", 26 "cranelift-entity", 27 "cranelift-bforest", 28 "cranelift-codegen-shared", 29 "cranelift-codegen-meta", 30 "cranelift-egraph", 31 "cranelift-control", 32 "cranelift-codegen", 33 "cranelift-reader", 34 "cranelift-serde", 35 "cranelift-module", 36 "cranelift-frontend", 37 "cranelift-native", 38 "cranelift-object", 39 "cranelift-interpreter", 40 "wasmtime-jit-icache-coherence", 41 "cranelift-jit", 42 "cranelift", 43 // wiggle 44 "wiggle-generate", 45 "wiggle-macro", 46 // winch 47 "winch", 48 // wasmtime 49 "wasmtime-asm-macros", 50 "wasmtime-versioned-export-macros", 51 "wasmtime-slab", 52 "wasmtime-component-util", 53 "wasmtime-wit-bindgen", 54 "wasmtime-component-macro", 55 "wasmtime-jit-debug", 56 "wasmtime-fiber", 57 "wasmtime-environ", 58 "wasmtime-wmemcheck", 59 "wasmtime-cranelift", 60 "wasmtime-cache", 61 "winch-codegen", 62 "wasmtime-winch", 63 "wasmtime", 64 // wasi-common/wiggle 65 "wiggle", 66 "wasi-common", 67 // other misc wasmtime crates 68 "wasmtime-wasi-io", 69 "wasmtime-wasi", 70 "wasmtime-wasi-http", 71 "wasmtime-wasi-nn", 72 "wasmtime-wasi-config", 73 "wasmtime-wasi-keyvalue", 74 "wasmtime-wasi-threads", 75 "wasmtime-wast", 76 "wasmtime-c-api-macros", 77 "wasmtime-c-api-impl", 78 "wasmtime-cli-flags", 79 "wasmtime-explorer", 80 "wasmtime-cli", 81 ]; 82 83 // Anything **not** mentioned in this array is required to have an `=a.b.c` 84 // dependency requirement on it to enable breaking api changes even in "patch" 85 // releases since everything not mentioned here is just an organizational detail 86 // that no one else should rely on. 87 const PUBLIC_CRATES: &[&str] = &[ 88 // these are actually public crates which we cannot break the API of in 89 // patch releases. 90 "wasmtime", 91 "wasmtime-wasi-io", 92 "wasmtime-wasi", 93 "wasmtime-wasi-http", 94 "wasmtime-wasi-nn", 95 "wasmtime-wasi-config", 96 "wasmtime-wasi-keyvalue", 97 "wasmtime-wasi-threads", 98 "wasmtime-cli", 99 // all cranelift crates are considered "public" in that they can't 100 // have breaking API changes in patch releases 101 "cranelift-entity", 102 "cranelift-bforest", 103 "cranelift-bitset", 104 "cranelift-codegen-shared", 105 "cranelift-codegen-meta", 106 "cranelift-egraph", 107 "cranelift-control", 108 "cranelift-codegen", 109 "cranelift-reader", 110 "cranelift-serde", 111 "cranelift-module", 112 "cranelift-frontend", 113 "cranelift-native", 114 "cranelift-object", 115 "cranelift-interpreter", 116 "cranelift", 117 "cranelift-jit", 118 // This is a dependency of cranelift crates and as a result can't break in 119 // patch releases as well 120 "wasmtime-types", 121 ]; 122 123 const C_HEADER_PATH: &str = "./crates/c-api/include/wasmtime.h"; 124 125 struct Workspace { 126 version: String, 127 } 128 129 struct Crate { 130 manifest: PathBuf, 131 name: String, 132 version: String, 133 publish: bool, 134 } 135 136 fn main() { 137 let mut crates = Vec::new(); 138 let root = read_crate(None, "./Cargo.toml".as_ref()); 139 let ws = Workspace { 140 version: root.version.clone(), 141 }; 142 crates.push(root); 143 find_crates("crates".as_ref(), &ws, &mut crates); 144 find_crates("cranelift".as_ref(), &ws, &mut crates); 145 find_crates("pulley".as_ref(), &ws, &mut crates); 146 find_crates("winch".as_ref(), &ws, &mut crates); 147 148 let pos = CRATES_TO_PUBLISH 149 .iter() 150 .enumerate() 151 .map(|(i, c)| (*c, i)) 152 .collect::<HashMap<_, _>>(); 153 crates.sort_by_key(|krate| pos.get(&krate.name[..])); 154 155 match &env::args().nth(1).expect("must have one argument")[..] { 156 name @ "bump" | name @ "bump-patch" => { 157 for krate in crates.iter() { 158 bump_version(&krate, &crates, name == "bump-patch"); 159 } 160 // update C API version in wasmtime.h 161 update_capi_version(); 162 // update the lock file 163 run_cmd(Command::new("cargo").arg("fetch")); 164 } 165 166 "publish" => { 167 // We have so many crates to publish we're frequently either 168 // rate-limited or we run into issues where crates can't publish 169 // successfully because they're waiting on the index entries of 170 // previously-published crates to propagate. This means we try to 171 // publish in a loop and we remove crates once they're successfully 172 // published. Failed-to-publish crates get enqueued for another try 173 // later on. 174 for _ in 0..10 { 175 crates.retain(|krate| !publish(krate)); 176 177 if crates.is_empty() { 178 break; 179 } 180 181 println!( 182 "{} crates failed to publish, waiting for a bit to retry", 183 crates.len(), 184 ); 185 thread::sleep(Duration::from_secs(40)); 186 } 187 188 assert!(crates.is_empty(), "failed to publish all crates"); 189 190 println!(""); 191 println!("==================================================================="); 192 println!(""); 193 println!("Don't forget to push a git tag for this release!"); 194 println!(""); 195 println!(" $ git tag vX.Y.Z"); 196 println!(" $ git push [email protected]:bytecodealliance/wasmtime.git vX.Y.Z"); 197 } 198 199 "verify" => { 200 verify(&crates); 201 } 202 203 s => panic!("unknown command: {}", s), 204 } 205 } 206 207 fn cmd_output(cmd: &mut Command) -> Output { 208 eprintln!("Running: `{:?}`", cmd); 209 match cmd.output() { 210 Ok(o) => o, 211 Err(e) => panic!("Failed to run `{:?}`: {}", cmd, e), 212 } 213 } 214 215 fn cmd_status(cmd: &mut Command) -> ExitStatus { 216 eprintln!("Running: `{:?}`", cmd); 217 match cmd.status() { 218 Ok(s) => s, 219 Err(e) => panic!("Failed to run `{:?}`: {}", cmd, e), 220 } 221 } 222 223 fn run_cmd(cmd: &mut Command) { 224 let status = cmd_status(cmd); 225 assert!( 226 status.success(), 227 "Command `{:?}` exited with failure status: {}", 228 cmd, 229 status 230 ); 231 } 232 233 fn find_crates(dir: &Path, ws: &Workspace, dst: &mut Vec<Crate>) { 234 if dir.join("Cargo.toml").exists() { 235 let krate = read_crate(Some(ws), &dir.join("Cargo.toml")); 236 if !krate.publish || CRATES_TO_PUBLISH.iter().any(|c| krate.name == *c) { 237 dst.push(krate); 238 } else { 239 panic!("failed to find {:?} in whitelist or blacklist", krate.name); 240 } 241 } 242 243 for entry in dir.read_dir().unwrap() { 244 let entry = entry.unwrap(); 245 if entry.file_type().unwrap().is_dir() { 246 find_crates(&entry.path(), ws, dst); 247 } 248 } 249 } 250 251 fn read_crate(ws: Option<&Workspace>, manifest: &Path) -> Crate { 252 let mut name = None; 253 let mut version = None; 254 let mut publish = true; 255 for line in fs::read_to_string(manifest).unwrap().lines() { 256 if name.is_none() && line.starts_with("name = \"") { 257 name = Some( 258 line.replace("name = \"", "") 259 .replace("\"", "") 260 .trim() 261 .to_string(), 262 ); 263 } 264 if version.is_none() && line.starts_with("version = \"") { 265 version = Some( 266 line.replace("version = \"", "") 267 .replace("\"", "") 268 .trim() 269 .to_string(), 270 ); 271 } 272 if let Some(ws) = ws { 273 if version.is_none() && line.starts_with("version.workspace = true") { 274 version = Some(ws.version.clone()); 275 } 276 } 277 if line.starts_with("publish = false") { 278 publish = false; 279 } 280 } 281 let name = name.unwrap(); 282 let version = version.unwrap(); 283 Crate { 284 manifest: manifest.to_path_buf(), 285 name, 286 version, 287 publish, 288 } 289 } 290 291 fn bump_version(krate: &Crate, crates: &[Crate], patch: bool) { 292 let contents = fs::read_to_string(&krate.manifest).unwrap(); 293 let next_version = |krate: &Crate| -> String { 294 if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { 295 bump(&krate.version, patch) 296 } else { 297 krate.version.clone() 298 } 299 }; 300 301 let mut new_manifest = String::new(); 302 let mut is_deps = false; 303 for line in contents.lines() { 304 let mut rewritten = false; 305 if !is_deps && line.starts_with("version =") { 306 if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { 307 println!( 308 "bump `{}` {} => {}", 309 krate.name, 310 krate.version, 311 next_version(krate), 312 ); 313 new_manifest.push_str(&line.replace(&krate.version, &next_version(krate))); 314 rewritten = true; 315 } 316 } 317 318 is_deps = if line.starts_with("[") { 319 line.contains("dependencies") 320 } else { 321 is_deps 322 }; 323 324 for other in crates { 325 // If `other` isn't a published crate then it's not going to get a 326 // bumped version so we don't need to update anything in the 327 // manifest. 328 if !other.publish { 329 continue; 330 } 331 if !is_deps || !line.starts_with(&format!("{} ", other.name)) { 332 continue; 333 } 334 if !line.contains(&other.version) { 335 if !line.contains("version =") || !krate.publish { 336 continue; 337 } 338 panic!( 339 "{:?} has a dep on {} but doesn't list version {}", 340 krate.manifest, other.name, other.version 341 ); 342 } 343 if krate.publish { 344 if PUBLIC_CRATES.contains(&other.name.as_str()) { 345 assert!( 346 !line.contains("\"="), 347 "{} should not have an exact version requirement on {}", 348 krate.name, 349 other.name 350 ); 351 } else { 352 assert!( 353 line.contains("\"="), 354 "{} should have an exact version requirement on {}", 355 krate.name, 356 other.name 357 ); 358 } 359 } 360 rewritten = true; 361 new_manifest.push_str(&line.replace(&other.version, &next_version(other))); 362 break; 363 } 364 if !rewritten { 365 new_manifest.push_str(line); 366 } 367 new_manifest.push_str("\n"); 368 } 369 fs::write(&krate.manifest, new_manifest).unwrap(); 370 } 371 372 fn update_capi_version() { 373 let version = read_crate(None, "./Cargo.toml".as_ref()).version; 374 375 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 376 let major = iter.next().expect("major version"); 377 let minor = iter.next().expect("minor version"); 378 let patch = iter.next().expect("patch version"); 379 380 let mut new_header = String::new(); 381 let contents = fs::read_to_string(C_HEADER_PATH).unwrap(); 382 for line in contents.lines() { 383 if line.starts_with("#define WASMTIME_VERSION \"") { 384 new_header.push_str(&format!("#define WASMTIME_VERSION \"{version}\"")); 385 } else if line.starts_with("#define WASMTIME_VERSION_MAJOR") { 386 new_header.push_str(&format!("#define WASMTIME_VERSION_MAJOR {major}")); 387 } else if line.starts_with("#define WASMTIME_VERSION_MINOR") { 388 new_header.push_str(&format!("#define WASMTIME_VERSION_MINOR {minor}")); 389 } else if line.starts_with("#define WASMTIME_VERSION_PATCH") { 390 new_header.push_str(&format!("#define WASMTIME_VERSION_PATCH {patch}")); 391 } else { 392 new_header.push_str(line); 393 } 394 new_header.push_str("\n"); 395 } 396 397 fs::write(&C_HEADER_PATH, new_header).unwrap(); 398 } 399 400 /// Performs a major version bump increment on the semver version `version`. 401 /// 402 /// This function will perform a semver-major-version bump on the `version` 403 /// specified. This is used to calculate the next version of a crate in this 404 /// repository since we're currently making major version bumps for all our 405 /// releases. This may end up getting tweaked as we stabilize crates and start 406 /// doing more minor/patch releases, but for now this should do the trick. 407 fn bump(version: &str, patch_bump: bool) -> String { 408 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 409 let major = iter.next().expect("major version"); 410 let minor = iter.next().expect("minor version"); 411 let patch = iter.next().expect("patch version"); 412 413 if patch_bump { 414 return format!("{}.{}.{}", major, minor, patch + 1); 415 } 416 if major != 0 { 417 format!("{}.0.0", major + 1) 418 } else if minor != 0 { 419 format!("0.{}.0", minor + 1) 420 } else { 421 format!("0.0.{}", patch + 1) 422 } 423 } 424 425 fn publish(krate: &Crate) -> bool { 426 if !CRATES_TO_PUBLISH.iter().any(|s| *s == krate.name) { 427 return true; 428 } 429 430 // First make sure the crate isn't already published at this version. This 431 // script may be re-run and there's no need to re-attempt previous work. 432 let output = cmd_output(Command::new("curl").arg(&format!( 433 "https://crates.io/api/v1/crates/{}/versions", 434 krate.name 435 ))); 436 if output.status.success() 437 && String::from_utf8_lossy(&output.stdout) 438 .contains(&format!("\"num\":\"{}\"", krate.version)) 439 { 440 println!( 441 "skip publish {} because {} is already published", 442 krate.name, krate.version, 443 ); 444 return true; 445 } 446 447 let status = cmd_status( 448 Command::new("cargo") 449 .arg("publish") 450 .current_dir(krate.manifest.parent().unwrap()) 451 .arg("--no-verify"), 452 ); 453 if !status.success() { 454 println!("FAIL: failed to publish `{}`: {}", krate.name, status); 455 return false; 456 } 457 458 // After we've published then make sure that the `wasmtime-publish` group is 459 // added to this crate for future publications. If it's already present 460 // though we can skip the `cargo owner` modification. 461 let output = cmd_output(Command::new("curl").arg(&format!( 462 "https://crates.io/api/v1/crates/{}/owners", 463 krate.name 464 ))); 465 if output.status.success() 466 && String::from_utf8_lossy(&output.stdout).contains("wasmtime-publish") 467 { 468 println!( 469 "wasmtime-publish already listed as an owner of {}", 470 krate.name 471 ); 472 return true; 473 } 474 475 // Note that the status is ignored here. This fails most of the time because 476 // the owner is already set and present, so we only want to add this to 477 // crates which haven't previously been published. 478 run_cmd( 479 Command::new("cargo") 480 .arg("owner") 481 .arg("-a") 482 .arg("github:bytecodealliance:wasmtime-publish") 483 .arg(&krate.name), 484 ); 485 486 true 487 } 488 489 // Verify the current tree is publish-able to crates.io. The intention here is 490 // that we'll run `cargo package` on everything which verifies the build as-if 491 // it were published to crates.io. This requires using an incrementally-built 492 // directory registry generated from `cargo vendor` because the versions 493 // referenced from `Cargo.toml` may not exist on crates.io. 494 fn verify(crates: &[Crate]) { 495 verify_capi(); 496 497 if Path::new(".cargo").exists() { 498 panic!( 499 "`.cargo` already exists on the file system, remove it and then run the script again" 500 ); 501 } 502 if Path::new("vendor").exists() { 503 panic!( 504 "`vendor` already exists on the file system, remove it and then run the script again" 505 ); 506 } 507 508 let vendor = cmd_output(Command::new("cargo").arg("vendor").stderr(Stdio::inherit())); 509 assert!(vendor.status.success()); 510 511 fs::create_dir_all(".cargo").unwrap(); 512 fs::write(".cargo/config.toml", vendor.stdout).unwrap(); 513 514 for krate in crates { 515 if !krate.publish { 516 continue; 517 } 518 verify_and_vendor(&krate); 519 } 520 521 fn verify_and_vendor(krate: &Crate) { 522 let mut cmd = Command::new("cargo"); 523 cmd.arg("package") 524 .arg("--manifest-path") 525 .arg(&krate.manifest) 526 .env("CARGO_TARGET_DIR", "./target"); 527 if krate.name.contains("wasi-nn") { 528 cmd.arg("--no-verify"); 529 } 530 run_cmd(&mut cmd); 531 run_cmd( 532 Command::new("tar") 533 .arg("xf") 534 .arg(format!( 535 "../target/package/{}-{}.crate", 536 krate.name, krate.version 537 )) 538 .current_dir("./vendor"), 539 ); 540 fs::write( 541 format!( 542 "./vendor/{}-{}/.cargo-checksum.json", 543 krate.name, krate.version 544 ), 545 "{\"files\":{}}", 546 ) 547 .unwrap(); 548 } 549 550 fn verify_capi() { 551 let version = read_crate(None, "./Cargo.toml".as_ref()).version; 552 553 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 554 let major = iter.next().expect("major version"); 555 let minor = iter.next().expect("minor version"); 556 let patch = iter.next().expect("patch version"); 557 558 let mut count = 0; 559 let contents = fs::read_to_string(C_HEADER_PATH).unwrap(); 560 for line in contents.lines() { 561 if line.starts_with(&format!("#define WASMTIME_VERSION \"{version}\"")) { 562 count += 1; 563 } else if line.starts_with(&format!("#define WASMTIME_VERSION_MAJOR {major}")) { 564 count += 1; 565 } else if line.starts_with(&format!("#define WASMTIME_VERSION_MINOR {minor}")) { 566 count += 1; 567 } else if line.starts_with(&format!("#define WASMTIME_VERSION_PATCH {patch}")) { 568 count += 1; 569 } 570 } 571 572 assert!( 573 count == 4, 574 "invalid version macros in {}, should match \"{}\"", 575 C_HEADER_PATH, 576 version 577 ); 578 } 579 } 580