1 //! Helper script to publish the wasmtime and cranelift suites of crates 2 //! 3 //! See documentation in `docs/contributing-release-process.md` for more 4 //! information, but in a nutshell: 5 //! 6 //! * `./publish bump` - bump crate versions in-tree 7 //! * `./publish verify` - verify crates can be published to crates.io 8 //! * `./publish publish` - actually publish crates to crates.io 9 10 use std::collections::HashMap; 11 use std::env; 12 use std::fs; 13 use std::path::{Path, PathBuf}; 14 use std::process::{Command, ExitStatus, Output, Stdio}; 15 use std::thread; 16 use std::time::Duration; 17 18 // note that this list must be topologically sorted by dependencies 19 const CRATES_TO_PUBLISH: &[&str] = &[ 20 // cranelift 21 "cranelift-isle", 22 "cranelift-entity", 23 "wasmtime-types", 24 "cranelift-bforest", 25 "cranelift-codegen-shared", 26 "cranelift-codegen-meta", 27 "cranelift-egraph", 28 "cranelift-control", 29 "cranelift-codegen", 30 "cranelift-reader", 31 "cranelift-serde", 32 "cranelift-module", 33 "cranelift-frontend", 34 "cranelift-wasm", 35 "cranelift-native", 36 "cranelift-object", 37 "cranelift-interpreter", 38 "cranelift", 39 "wasmtime-jit-icache-coherence", 40 "cranelift-jit", 41 // wiggle 42 "wiggle-generate", 43 "wiggle-macro", 44 // winch 45 "winch", 46 // wasmtime 47 "wasmtime-asm-macros", 48 "wasmtime-versioned-export-macros", 49 "wasmtime-slab", 50 "wasmtime-component-util", 51 "wasmtime-wit-bindgen", 52 "wasmtime-component-macro", 53 "wasmtime-jit-debug", 54 "wasmtime-fiber", 55 "wasmtime-environ", 56 "wasmtime-wmemcheck", 57 "wasmtime-cranelift", 58 "wasmtime-cache", 59 "winch-codegen", 60 "wasmtime-winch", 61 "wasmtime", 62 // wasi-common/wiggle 63 "wiggle", 64 "wasi-common", 65 // other misc wasmtime crates 66 "wasmtime-wasi", 67 "wasmtime-wasi-http", 68 "wasmtime-wasi-nn", 69 "wasmtime-wasi-threads", 70 "wasmtime-wast", 71 "wasmtime-c-api-macros", 72 "wasmtime-c-api-impl", 73 "wasmtime-cli-flags", 74 "wasmtime-explorer", 75 "wasmtime-cli", 76 ]; 77 78 // Anything **not** mentioned in this array is required to have an `=a.b.c` 79 // dependency requirement on it to enable breaking api changes even in "patch" 80 // releases since everything not mentioned here is just an organizational detail 81 // that no one else should rely on. 82 const PUBLIC_CRATES: &[&str] = &[ 83 // these are actually public crates which we cannot break the API of in 84 // patch releases. 85 "wasmtime", 86 "wasmtime-wasi", 87 "wasmtime-wasi-nn", 88 "wasmtime-wasi-threads", 89 "wasmtime-cli", 90 // all cranelift crates are considered "public" in that they can't 91 // have breaking API changes in patch releases 92 "cranelift-entity", 93 "cranelift-bforest", 94 "cranelift-codegen-shared", 95 "cranelift-codegen-meta", 96 "cranelift-egraph", 97 "cranelift-control", 98 "cranelift-codegen", 99 "cranelift-reader", 100 "cranelift-serde", 101 "cranelift-module", 102 "cranelift-frontend", 103 "cranelift-wasm", 104 "cranelift-native", 105 "cranelift-object", 106 "cranelift-interpreter", 107 "cranelift", 108 "cranelift-jit", 109 // This is a dependency of cranelift crates and as a result can't break in 110 // patch releases as well 111 "wasmtime-types", 112 ]; 113 114 const C_HEADER_PATH: &str = "./crates/c-api/include/wasmtime.h"; 115 116 struct Workspace { 117 version: String, 118 } 119 120 struct Crate { 121 manifest: PathBuf, 122 name: String, 123 version: String, 124 publish: bool, 125 } 126 127 fn main() { 128 let mut crates = Vec::new(); 129 let root = read_crate(None, "./Cargo.toml".as_ref()); 130 let ws = Workspace { 131 version: root.version.clone(), 132 }; 133 crates.push(root); 134 find_crates("crates".as_ref(), &ws, &mut crates); 135 find_crates("cranelift".as_ref(), &ws, &mut crates); 136 find_crates("winch".as_ref(), &ws, &mut crates); 137 138 let pos = CRATES_TO_PUBLISH 139 .iter() 140 .enumerate() 141 .map(|(i, c)| (*c, i)) 142 .collect::<HashMap<_, _>>(); 143 crates.sort_by_key(|krate| pos.get(&krate.name[..])); 144 145 match &env::args().nth(1).expect("must have one argument")[..] { 146 name @ "bump" | name @ "bump-patch" => { 147 for krate in crates.iter() { 148 bump_version(&krate, &crates, name == "bump-patch"); 149 } 150 // update C API version in wasmtime.h 151 update_capi_version(); 152 // update the lock file 153 run_cmd(Command::new("cargo").arg("fetch")); 154 } 155 156 "publish" => { 157 // We have so many crates to publish we're frequently either 158 // rate-limited or we run into issues where crates can't publish 159 // successfully because they're waiting on the index entries of 160 // previously-published crates to propagate. This means we try to 161 // publish in a loop and we remove crates once they're successfully 162 // published. Failed-to-publish crates get enqueued for another try 163 // later on. 164 for _ in 0..10 { 165 crates.retain(|krate| !publish(krate)); 166 167 if crates.is_empty() { 168 break; 169 } 170 171 println!( 172 "{} crates failed to publish, waiting for a bit to retry", 173 crates.len(), 174 ); 175 thread::sleep(Duration::from_secs(40)); 176 } 177 178 assert!(crates.is_empty(), "failed to publish all crates"); 179 180 println!(""); 181 println!("==================================================================="); 182 println!(""); 183 println!("Don't forget to push a git tag for this release!"); 184 println!(""); 185 println!(" $ git tag vX.Y.Z"); 186 println!(" $ git push [email protected]:bytecodealliance/wasmtime.git vX.Y.Z"); 187 } 188 189 "verify" => { 190 verify(&crates); 191 } 192 193 s => panic!("unknown command: {}", s), 194 } 195 } 196 197 fn cmd_output(cmd: &mut Command) -> Output { 198 eprintln!("Running: `{:?}`", cmd); 199 match cmd.output() { 200 Ok(o) => o, 201 Err(e) => panic!("Failed to run `{:?}`: {}", cmd, e), 202 } 203 } 204 205 fn cmd_status(cmd: &mut Command) -> ExitStatus { 206 eprintln!("Running: `{:?}`", cmd); 207 match cmd.status() { 208 Ok(s) => s, 209 Err(e) => panic!("Failed to run `{:?}`: {}", cmd, e), 210 } 211 } 212 213 fn run_cmd(cmd: &mut Command) { 214 let status = cmd_status(cmd); 215 assert!( 216 status.success(), 217 "Command `{:?}` exited with failure status: {}", 218 cmd, 219 status 220 ); 221 } 222 223 fn find_crates(dir: &Path, ws: &Workspace, dst: &mut Vec<Crate>) { 224 if dir.join("Cargo.toml").exists() { 225 let krate = read_crate(Some(ws), &dir.join("Cargo.toml")); 226 if !krate.publish || CRATES_TO_PUBLISH.iter().any(|c| krate.name == *c) { 227 dst.push(krate); 228 } else { 229 panic!("failed to find {:?} in whitelist or blacklist", krate.name); 230 } 231 } 232 233 for entry in dir.read_dir().unwrap() { 234 let entry = entry.unwrap(); 235 if entry.file_type().unwrap().is_dir() { 236 find_crates(&entry.path(), ws, dst); 237 } 238 } 239 } 240 241 fn read_crate(ws: Option<&Workspace>, manifest: &Path) -> Crate { 242 let mut name = None; 243 let mut version = None; 244 let mut publish = true; 245 for line in fs::read_to_string(manifest).unwrap().lines() { 246 if name.is_none() && line.starts_with("name = \"") { 247 name = Some( 248 line.replace("name = \"", "") 249 .replace("\"", "") 250 .trim() 251 .to_string(), 252 ); 253 } 254 if version.is_none() && line.starts_with("version = \"") { 255 version = Some( 256 line.replace("version = \"", "") 257 .replace("\"", "") 258 .trim() 259 .to_string(), 260 ); 261 } 262 if let Some(ws) = ws { 263 if version.is_none() && line.starts_with("version.workspace = true") { 264 version = Some(ws.version.clone()); 265 } 266 } 267 if line.starts_with("publish = false") { 268 publish = false; 269 } 270 } 271 let name = name.unwrap(); 272 let version = version.unwrap(); 273 Crate { 274 manifest: manifest.to_path_buf(), 275 name, 276 version, 277 publish, 278 } 279 } 280 281 fn bump_version(krate: &Crate, crates: &[Crate], patch: bool) { 282 let contents = fs::read_to_string(&krate.manifest).unwrap(); 283 let next_version = |krate: &Crate| -> String { 284 if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { 285 bump(&krate.version, patch) 286 } else { 287 krate.version.clone() 288 } 289 }; 290 291 let mut new_manifest = String::new(); 292 let mut is_deps = false; 293 for line in contents.lines() { 294 let mut rewritten = false; 295 if !is_deps && line.starts_with("version =") { 296 if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { 297 println!( 298 "bump `{}` {} => {}", 299 krate.name, 300 krate.version, 301 next_version(krate), 302 ); 303 new_manifest.push_str(&line.replace(&krate.version, &next_version(krate))); 304 rewritten = true; 305 } 306 } 307 308 is_deps = if line.starts_with("[") { 309 line.contains("dependencies") 310 } else { 311 is_deps 312 }; 313 314 for other in crates { 315 // If `other` isn't a published crate then it's not going to get a 316 // bumped version so we don't need to update anything in the 317 // manifest. 318 if !other.publish { 319 continue; 320 } 321 if !is_deps || !line.starts_with(&format!("{} ", other.name)) { 322 continue; 323 } 324 if !line.contains(&other.version) { 325 if !line.contains("version =") || !krate.publish { 326 continue; 327 } 328 panic!( 329 "{:?} has a dep on {} but doesn't list version {}", 330 krate.manifest, other.name, other.version 331 ); 332 } 333 if krate.publish { 334 if PUBLIC_CRATES.contains(&other.name.as_str()) { 335 assert!( 336 !line.contains("\"="), 337 "{} should not have an exact version requirement on {}", 338 krate.name, 339 other.name 340 ); 341 } else { 342 assert!( 343 line.contains("\"="), 344 "{} should have an exact version requirement on {}", 345 krate.name, 346 other.name 347 ); 348 } 349 } 350 rewritten = true; 351 new_manifest.push_str(&line.replace(&other.version, &next_version(other))); 352 break; 353 } 354 if !rewritten { 355 new_manifest.push_str(line); 356 } 357 new_manifest.push_str("\n"); 358 } 359 fs::write(&krate.manifest, new_manifest).unwrap(); 360 } 361 362 fn update_capi_version() { 363 let version = read_crate(None, "./Cargo.toml".as_ref()).version; 364 365 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 366 let major = iter.next().expect("major version"); 367 let minor = iter.next().expect("minor version"); 368 let patch = iter.next().expect("patch version"); 369 370 let mut new_header = String::new(); 371 let contents = fs::read_to_string(C_HEADER_PATH).unwrap(); 372 for line in contents.lines() { 373 if line.starts_with("#define WASMTIME_VERSION \"") { 374 new_header.push_str(&format!("#define WASMTIME_VERSION \"{version}\"")); 375 } else if line.starts_with("#define WASMTIME_VERSION_MAJOR") { 376 new_header.push_str(&format!("#define WASMTIME_VERSION_MAJOR {major}")); 377 } else if line.starts_with("#define WASMTIME_VERSION_MINOR") { 378 new_header.push_str(&format!("#define WASMTIME_VERSION_MINOR {minor}")); 379 } else if line.starts_with("#define WASMTIME_VERSION_PATCH") { 380 new_header.push_str(&format!("#define WASMTIME_VERSION_PATCH {patch}")); 381 } else { 382 new_header.push_str(line); 383 } 384 new_header.push_str("\n"); 385 } 386 387 fs::write(&C_HEADER_PATH, new_header).unwrap(); 388 } 389 390 /// Performs a major version bump increment on the semver version `version`. 391 /// 392 /// This function will perform a semver-major-version bump on the `version` 393 /// specified. This is used to calculate the next version of a crate in this 394 /// repository since we're currently making major version bumps for all our 395 /// releases. This may end up getting tweaked as we stabilize crates and start 396 /// doing more minor/patch releases, but for now this should do the trick. 397 fn bump(version: &str, patch_bump: bool) -> String { 398 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 399 let major = iter.next().expect("major version"); 400 let minor = iter.next().expect("minor version"); 401 let patch = iter.next().expect("patch version"); 402 403 if patch_bump { 404 return format!("{}.{}.{}", major, minor, patch + 1); 405 } 406 if major != 0 { 407 format!("{}.0.0", major + 1) 408 } else if minor != 0 { 409 format!("0.{}.0", minor + 1) 410 } else { 411 format!("0.0.{}", patch + 1) 412 } 413 } 414 415 fn publish(krate: &Crate) -> bool { 416 if !CRATES_TO_PUBLISH.iter().any(|s| *s == krate.name) { 417 return true; 418 } 419 420 // First make sure the crate isn't already published at this version. This 421 // script may be re-run and there's no need to re-attempt previous work. 422 let output = cmd_output( 423 Command::new("curl").arg(&format!("https://crates.io/api/v1/crates/{}", krate.name)), 424 ); 425 if output.status.success() 426 && String::from_utf8_lossy(&output.stdout) 427 .contains(&format!("\"newest_version\":\"{}\"", krate.version)) 428 { 429 println!( 430 "skip publish {} because {} is latest version", 431 krate.name, krate.version, 432 ); 433 return true; 434 } 435 436 let status = cmd_status( 437 Command::new("cargo") 438 .arg("publish") 439 .current_dir(krate.manifest.parent().unwrap()) 440 .arg("--no-verify"), 441 ); 442 if !status.success() { 443 println!("FAIL: failed to publish `{}`: {}", krate.name, status); 444 return false; 445 } 446 447 // After we've published then make sure that the `wasmtime-publish` group is 448 // added to this crate for future publications. If it's already present 449 // though we can skip the `cargo owner` modification. 450 let output = cmd_output(Command::new("curl").arg(&format!( 451 "https://crates.io/api/v1/crates/{}/owners", 452 krate.name 453 ))); 454 if output.status.success() 455 && String::from_utf8_lossy(&output.stdout).contains("wasmtime-publish") 456 { 457 println!( 458 "wasmtime-publish already listed as an owner of {}", 459 krate.name 460 ); 461 return true; 462 } 463 464 // Note that the status is ignored here. This fails most of the time because 465 // the owner is already set and present, so we only want to add this to 466 // crates which haven't previously been published. 467 run_cmd( 468 Command::new("cargo") 469 .arg("owner") 470 .arg("-a") 471 .arg("github:bytecodealliance:wasmtime-publish") 472 .arg(&krate.name), 473 ); 474 475 true 476 } 477 478 // Verify the current tree is publish-able to crates.io. The intention here is 479 // that we'll run `cargo package` on everything which verifies the build as-if 480 // it were published to crates.io. This requires using an incrementally-built 481 // directory registry generated from `cargo vendor` because the versions 482 // referenced from `Cargo.toml` may not exist on crates.io. 483 fn verify(crates: &[Crate]) { 484 verify_capi(); 485 486 drop(fs::remove_dir_all(".cargo")); 487 drop(fs::remove_dir_all("vendor")); 488 let vendor = cmd_output(Command::new("cargo").arg("vendor").stderr(Stdio::inherit())); 489 assert!(vendor.status.success()); 490 491 fs::create_dir_all(".cargo").unwrap(); 492 fs::write(".cargo/config.toml", vendor.stdout).unwrap(); 493 494 for krate in crates { 495 if !krate.publish { 496 continue; 497 } 498 verify_and_vendor(&krate); 499 } 500 501 fn verify_and_vendor(krate: &Crate) { 502 let mut cmd = Command::new("cargo"); 503 cmd.arg("package") 504 .arg("--manifest-path") 505 .arg(&krate.manifest) 506 .env("CARGO_TARGET_DIR", "./target"); 507 if krate.name.contains("wasi-nn") { 508 cmd.arg("--no-verify"); 509 } 510 run_cmd(&mut cmd); 511 run_cmd( 512 Command::new("tar") 513 .arg("xf") 514 .arg(format!( 515 "../target/package/{}-{}.crate", 516 krate.name, krate.version 517 )) 518 .current_dir("./vendor"), 519 ); 520 fs::write( 521 format!( 522 "./vendor/{}-{}/.cargo-checksum.json", 523 krate.name, krate.version 524 ), 525 "{\"files\":{}}", 526 ) 527 .unwrap(); 528 } 529 530 fn verify_capi() { 531 let version = read_crate(None, "./Cargo.toml".as_ref()).version; 532 533 let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap()); 534 let major = iter.next().expect("major version"); 535 let minor = iter.next().expect("minor version"); 536 let patch = iter.next().expect("patch version"); 537 538 let mut count = 0; 539 let contents = fs::read_to_string(C_HEADER_PATH).unwrap(); 540 for line in contents.lines() { 541 if line.starts_with(&format!("#define WASMTIME_VERSION \"{version}\"")) { 542 count += 1; 543 } else if line.starts_with(&format!("#define WASMTIME_VERSION_MAJOR {major}")) { 544 count += 1; 545 } else if line.starts_with(&format!("#define WASMTIME_VERSION_MINOR {minor}")) { 546 count += 1; 547 } else if line.starts_with(&format!("#define WASMTIME_VERSION_PATCH {patch}")) { 548 count += 1; 549 } 550 } 551 552 assert!( 553 count == 4, 554 "invalid version macros in {}, should match \"{}\"", 555 C_HEADER_PATH, 556 version 557 ); 558 } 559 } 560