From 9307bb2a8820b80d073ab42e64b55d2d3bb27725 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 1 Feb 2023 21:47:30 -0600 Subject: [PATCH] Download beta compiler toolchain in bootstrap if it doesn't yet exist This is needed for when the shell scripts bypass python altogether and run the downloaded bootstrap directly --- src/bootstrap/config.rs | 35 ++++++++++---------- src/bootstrap/download.rs | 67 ++++++++++++++++++++++++++++++++------- src/bootstrap/flags.rs | 3 +- src/bootstrap/test.rs | 2 +- 4 files changed, 76 insertions(+), 31 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 165502b0a41d8..7bd5f33b58c23 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -55,8 +55,7 @@ pub enum DryRun { /// filled out from the decoded forms of the structs below. For documentation /// each field, see the corresponding fields in /// `config.toml.example`. -#[derive(Default)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Default)] pub struct Config { pub changelog_seen: Option, pub ccache: Option, @@ -217,27 +216,33 @@ pub struct Config { pub reuse: Option, pub cargo_native_static: bool, pub configure_args: Vec, + pub out: PathBuf, + pub rust_info: channel::GitInfo, // These are either the stage0 downloaded binaries or the locally installed ones. pub initial_cargo: PathBuf, pub initial_rustc: PathBuf, + #[cfg(not(test))] initial_rustfmt: RefCell, #[cfg(test)] pub initial_rustfmt: RefCell, - pub out: PathBuf, - pub rust_info: channel::GitInfo, } -#[derive(Default, Deserialize)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Default, Deserialize)] pub struct Stage0Metadata { + pub compiler: CompilerMetadata, pub config: Stage0Config, pub checksums_sha256: HashMap, pub rustfmt: Option, } -#[derive(Default, Deserialize)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Default, Deserialize)] +pub struct CompilerMetadata { + pub date: String, + pub version: String, +} + +#[derive(Clone, Default, Deserialize)] pub struct Stage0Config { pub dist_server: String, pub artifacts_server: String, @@ -245,8 +250,7 @@ pub struct Stage0Config { pub git_merge_commit_email: String, pub nightly_branch: String, } -#[derive(Default, Deserialize)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Default, Deserialize)] pub struct RustfmtMetadata { pub date: String, pub version: String, @@ -422,8 +426,7 @@ impl PartialEq<&str> for TargetSelection { } /// Per-target configuration stored in the global configuration structure. -#[derive(Default)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Default)] pub struct Target { /// Some(path to llvm-config) if using an external LLVM. pub llvm_config: Option, @@ -979,10 +982,10 @@ impl Config { config.out = crate::util::absolute(&config.out); } - config.initial_rustc = build - .rustc - .map(PathBuf::from) - .unwrap_or_else(|| config.out.join(config.build.triple).join("stage0/bin/rustc")); + config.initial_rustc = build.rustc.map(PathBuf::from).unwrap_or_else(|| { + config.download_beta_toolchain(); + config.out.join(config.build.triple).join("stage0/bin/rustc") + }); config.initial_cargo = build .cargo .map(PathBuf::from) diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index bd67978a7662e..2b135675f32c6 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -361,26 +361,69 @@ impl Config { pub(crate) fn download_ci_rustc(&self, commit: &str) { self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); + let version = self.artifact_version_part(commit); + // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the + // `rustc_private` crates for tools. + let extra_components = ["rustc-dev"]; + + self.download_toolchain( + &version, + "ci-rustc", + commit, + &extra_components, + Self::download_ci_component, + ); + } + + pub(crate) fn download_beta_toolchain(&self) { + self.verbose(&format!("downloading stage0 beta artifacts")); + + let date = &self.stage0_metadata.compiler.date; + let version = &self.stage0_metadata.compiler.version; + let extra_components = ["cargo"]; + + let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { + config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") + }; + + self.download_toolchain( + version, + "stage0", + date, + &extra_components, + download_beta_component, + ); + } + + fn download_toolchain( + &self, + channel: &str, + sysroot: &str, + stamp_key: &str, + extra_components: &[&str], + download_component: fn(&Config, String, &str, &str), + ) { let host = self.build.triple; - let bin_root = self.out.join(host).join("ci-rustc"); + let bin_root = self.out.join(host).join(sysroot); let rustc_stamp = bin_root.join(".rustc-stamp"); - if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) + if !bin_root.join("bin").join("rustc").exists() + || program_out_of_date(&rustc_stamp, stamp_key) { if bin_root.exists() { t!(fs::remove_dir_all(&bin_root)); } - let filename = format!("rust-std-{version}-{host}.tar.xz"); + let filename = format!("rust-std-{channel}-{host}.tar.xz"); let pattern = format!("rust-std-{host}"); - self.download_ci_component(filename, &pattern, commit); - let filename = format!("rustc-{version}-{host}.tar.xz"); - self.download_ci_component(filename, "rustc", commit); - // download-rustc doesn't need its own cargo, it can just use beta's. - let filename = format!("rustc-dev-{version}-{host}.tar.xz"); - self.download_ci_component(filename, "rustc-dev", commit); - let filename = format!("rust-src-{version}.tar.xz"); - self.download_ci_component(filename, "rust-src", commit); + download_component(self, filename, &pattern, stamp_key); + let filename = format!("rustc-{channel}-{host}.tar.xz"); + download_component(self, filename, "rustc", stamp_key); + + for component in extra_components { + let filename = format!("{component}-{channel}-{host}.tar.xz"); + download_component(self, filename, component, stamp_key); + } if self.should_fix_bins_and_dylibs() { self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); @@ -397,7 +440,7 @@ impl Config { } } - t!(fs::write(rustc_stamp, commit)); + t!(fs::write(rustc_stamp, stamp_key)); } } diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 52c3dc0bf7591..af2faa72344cc 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -82,8 +82,7 @@ pub struct Flags { pub llvm_bolt_profile_use: Option, } -#[derive(Debug)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Debug)] pub enum Subcommand { Build { paths: Vec, diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 6078e39ac9d3b..41a65296ad9b7 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1120,7 +1120,7 @@ impl Step for Tidy { if builder.config.channel == "dev" || builder.config.channel == "nightly" { builder.info("fmt check"); if builder.initial_rustfmt().is_none() { - let inferred_rustfmt_dir = builder.config.initial_rustc.parent().unwrap(); + let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); eprintln!( "\ error: no `rustfmt` binary found in {PATH}