From ea813a0f7f7cd069a7262269b66564702312c275 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Thu, 17 Nov 2022 00:18:56 -0500 Subject: [PATCH 1/5] [wip] Add bootstrap-shim and package it on nightly - Pass `dist bootstrap-shim` explicitly in CI This makes it possible to run `dist bootstrap` without also building bootstrap-shim, and more importantly works around a bug where currently two steps aren't allowed to have the same path. - Add `check::BootstrapShim` - [wip] start unifying parsing for Config and MinimalConfig --- src/bootstrap/Cargo.lock | 16 + src/bootstrap/Cargo.toml | 6 + src/bootstrap/bin/bootstrap-shim.rs | 29 ++ src/bootstrap/builder.rs | 4 +- src/bootstrap/check.rs | 2 + src/bootstrap/config.rs | 316 ++------------ src/bootstrap/dist.rs | 35 ++ src/bootstrap/download.rs | 202 +++++---- src/bootstrap/flags.rs | 3 +- src/bootstrap/lib.rs | 2 + src/bootstrap/min_config.rs | 393 ++++++++++++++++++ .../host-x86_64/dist-x86_64-linux/Dockerfile | 2 +- src/ci/github-actions/ci.yml | 20 +- src/tools/build-manifest/src/main.rs | 6 +- src/tools/build-manifest/src/versions.rs | 3 + src/tools/x/src/main.rs | 2 +- 16 files changed, 675 insertions(+), 366 deletions(-) create mode 100644 src/bootstrap/bin/bootstrap-shim.rs create mode 100644 src/bootstrap/min_config.rs diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index 9420c4fec5fa4..5f0f2c333d54e 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -49,6 +49,7 @@ dependencies = [ "cmake", "fd-lock", "filetime", + "getopts", "hex", "ignore", "is-terminal", @@ -317,6 +318,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + [[package]] name = "globset" version = "0.4.8" @@ -778,6 +788,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "version_check" version = "0.9.4" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 746c8dcfce0cd..170205f1d5c9e 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -14,6 +14,11 @@ name = "bootstrap" path = "bin/main.rs" test = false +[[bin]] +name = "bootstrap-shim" +path = "bin/bootstrap-shim.rs" +test = false + [[bin]] name = "rustc" path = "bin/rustc.rs" @@ -35,6 +40,7 @@ build_helper = { path = "../tools/build_helper" } cmake = "0.1.38" filetime = "0.2" cc = "1.0.69" +getopts = "0.2" libc = "0.2" hex = "0.4" object = { version = "0.29.0", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] } diff --git a/src/bootstrap/bin/bootstrap-shim.rs b/src/bootstrap/bin/bootstrap-shim.rs new file mode 100644 index 0000000000000..d0d044d2373a8 --- /dev/null +++ b/src/bootstrap/bin/bootstrap-shim.rs @@ -0,0 +1,29 @@ +use std::{env, process::Command}; + +use bootstrap::{t, MinimalConfig}; + +#[path = "../../../src/tools/x/src/main.rs"] +mod run_python; + +fn main() { + let args = env::args().skip(1).collect::>(); + let mut opts = getopts::Options::new(); + opts.optopt("", "config", "TOML configuration file for build", "FILE"); + let matches = t!(opts.parse(args)); + + // If there are no untracked changes to bootstrap, download it from CI. + // Otherwise, build it from source. Use python to build to avoid duplicating the code between python and rust. + let config = MinimalConfig::parse(t!(matches.opt_get("config"))); + let bootstrap_bin = if let Some(commit) = last_modified_bootstrap_commit(&config) { + config.download_bootstrap(&commit) + } else { + return run_python::main(); + }; + + let args: Vec<_> = std::env::args().skip(1).collect(); + Command::new(bootstrap_bin).args(args).status().expect("failed to spawn bootstrap binairy"); +} + +fn last_modified_bootstrap_commit(config: &MinimalConfig) -> Option { + config.last_modified_commit(&["src/bootstrap"], "download-bootstrap", true) +} diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 5c37fab547070..18b5b8146bea3 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -703,7 +703,8 @@ impl<'a> Builder<'a> { check::Rls, check::RustAnalyzer, check::Rustfmt, - check::Bootstrap + check::Bootstrap, + check::BootstrapShim, ), Kind::Test => describe!( crate::toolstate::ToolStateCheck, @@ -808,6 +809,7 @@ impl<'a> Builder<'a> { dist::LlvmTools, dist::RustDev, dist::Bootstrap, + dist::BootstrapShim, dist::Extended, // It seems that PlainSourceTarball somehow changes how some of the tools // perceive their dependencies (see #93033) which would invalidate fingerprints diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index b11be96cefe62..2271034a13244 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -448,7 +448,9 @@ tool_check_step!(Rls, "src/tools/rls", SourceType::InTree); tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree); tool_check_step!(MiroptTestTools, "src/tools/miropt-test-tools", SourceType::InTree); +// FIXME: currently these are marked as ToolRustc, but they should be ToolBootstrap instead to avoid having to build the compiler first tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false); +tool_check_step!(BootstrapShim, "src/bootstrap/bin/bootstrap-shim", SourceType::InTree, false); /// Cargo's output path for the standard library in a given stage, compiled /// by a particular compiler for the specified target. diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 710c8b52194b4..01ec697fb353f 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -10,8 +10,8 @@ use std::cell::{Cell, RefCell}; use std::cmp; use std::collections::{HashMap, HashSet}; use std::env; -use std::fmt; use std::fs; +use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; use std::process::Command; use std::str::FromStr; @@ -22,12 +22,16 @@ use crate::cc_detect::{ndk_compiler, Language}; use crate::channel::{self, GitInfo}; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags, Warnings}; +use crate::min_config::get_toml; use crate::util::{exe, output, t}; +use crate::MinimalConfig; use once_cell::sync::OnceCell; use semver::Version; use serde::{Deserialize, Deserializer}; use serde_derive::Deserialize; +pub use crate::min_config::{DryRun, Stage0Metadata, TargetSelection, TargetSelectionList}; + macro_rules! check_ci_llvm { ($name:expr) => { assert!( @@ -38,17 +42,6 @@ macro_rules! check_ci_llvm { }; } -#[derive(Clone, Default)] -pub enum DryRun { - /// This isn't a dry run. - #[default] - Disabled, - /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. - SelfCheck, - /// This is a dry run enabled by the `--dry-run` flag. - UserSelected, -} - /// Global configuration for the entire build and/or bootstrap. /// /// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. @@ -63,7 +56,6 @@ pub struct Config { pub ccache: Option, /// Call Build::ninja() instead of this. pub ninja_in_file: bool, - pub verbose: usize, pub submodules: Option, pub compiler_docs: bool, pub library_docs_private_items: bool, @@ -84,8 +76,6 @@ pub struct Config { pub json_output: bool, pub test_compare_mode: bool, pub color: Color, - pub patch_binaries_for_nix: bool, - pub stage0_metadata: Stage0Metadata, pub stdout_is_tty: bool, pub stderr_is_tty: bool, @@ -94,9 +84,6 @@ pub struct Config { pub stage: u32, pub keep_stage: Vec, pub keep_stage_std: Vec, - pub src: PathBuf, - /// defaults to `config.toml` - pub config: Option, pub jobs: Option, pub cmd: Subcommand, pub incremental: bool, @@ -183,7 +170,6 @@ pub struct Config { pub llvm_bolt_profile_generate: bool, pub llvm_bolt_profile_use: Option, - pub build: TargetSelection, pub hosts: Vec, pub targets: Vec, pub local_rebuild: bool, @@ -227,7 +213,6 @@ pub struct Config { pub reuse: Option, pub cargo_native_static: bool, pub configure_args: Vec, - pub out: PathBuf, pub rust_info: channel::GitInfo, // These are either the stage0 downloaded binaries or the locally installed ones. @@ -240,33 +225,24 @@ pub struct Config { pub initial_rustfmt: RefCell, pub paths: Vec, -} -#[derive(Default, Deserialize, Clone)] -pub struct Stage0Metadata { - pub compiler: CompilerMetadata, - pub config: Stage0Config, - pub checksums_sha256: HashMap, - pub rustfmt: Option, -} -#[derive(Default, Deserialize, Clone)] -pub struct CompilerMetadata { - pub date: String, - pub version: String, + #[cfg(test)] + pub minimal_config: MinimalConfig, + #[cfg(not(test))] + minimal_config: MinimalConfig, } -#[derive(Default, Deserialize, Clone)] -pub struct Stage0Config { - pub dist_server: String, - pub artifacts_server: String, - pub artifacts_with_llvm_assertions_server: String, - pub git_merge_commit_email: String, - pub nightly_branch: String, +impl Deref for Config { + type Target = MinimalConfig; + fn deref(&self) -> &Self::Target { + &self.minimal_config + } } -#[derive(Default, Deserialize, Clone)] -pub struct RustfmtMetadata { - pub date: String, - pub version: String, + +impl DerefMut for Config { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.minimal_config + } } #[derive(Clone, Debug)] @@ -373,83 +349,6 @@ impl std::str::FromStr for RustcLto { } } -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct TargetSelection { - pub triple: Interned, - file: Option>, -} - -/// Newtype over `Vec` so we can implement custom parsing logic -#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] -pub struct TargetSelectionList(Vec); - -pub fn target_selection_list(s: &str) -> Result { - Ok(TargetSelectionList( - s.split(",").filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), - )) -} - -impl TargetSelection { - pub fn from_user(selection: &str) -> Self { - let path = Path::new(selection); - - let (triple, file) = if path.exists() { - let triple = path - .file_stem() - .expect("Target specification file has no file stem") - .to_str() - .expect("Target specification file stem is not UTF-8"); - - (triple, Some(selection)) - } else { - (selection, None) - }; - - let triple = INTERNER.intern_str(triple); - let file = file.map(|f| INTERNER.intern_str(f)); - - Self { triple, file } - } - - pub fn rustc_target_arg(&self) -> &str { - self.file.as_ref().unwrap_or(&self.triple) - } - - pub fn contains(&self, needle: &str) -> bool { - self.triple.contains(needle) - } - - pub fn starts_with(&self, needle: &str) -> bool { - self.triple.starts_with(needle) - } - - pub fn ends_with(&self, needle: &str) -> bool { - self.triple.ends_with(needle) - } -} - -impl fmt::Display for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.triple)?; - if let Some(file) = self.file { - write!(f, "({})", file)?; - } - Ok(()) - } -} - -impl fmt::Debug for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self) - } -} - -impl PartialEq<&str> for TargetSelection { - fn eq(&self, other: &&str) -> bool { - self.triple == *other - } -} - /// Per-target configuration stored in the global configuration structure. #[derive(Default, Clone)] pub struct Target { @@ -862,32 +761,32 @@ impl Config { config } - pub fn parse(args: &[String]) -> Config { - #[cfg(test)] - let get_toml = |_: &_| TomlConfig::default(); - #[cfg(not(test))] - let get_toml = |file: &Path| { - let contents = - t!(fs::read_to_string(file), format!("config file {} not found", file.display())); - // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of - // TomlConfig and sub types to be monomorphized 5x by toml. - match toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - { - Ok(table) => table, - Err(err) => { - eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err); - crate::detail_exit(2); - } + pub(crate) fn test_args(&self) -> Vec<&str> { + let mut test_args = match self.cmd { + Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => { + test_args.iter().flat_map(|s| s.split_whitespace()).collect() } + _ => vec![], }; + test_args.extend(self.free_args.iter().map(|s| s.as_str())); + test_args + } - Self::parse_inner(args, get_toml) + pub(crate) fn args(&self) -> Vec<&str> { + let mut args = match self.cmd { + Subcommand::Run { ref args, .. } => { + args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + }; + args.extend(self.free_args.iter().map(|s| s.as_str())); + args } - fn parse_inner<'a>(args: &[String], get_toml: impl 'a + Fn(&Path) -> TomlConfig) -> Config { + pub fn parse(args: &[String]) -> Config { let mut flags = Flags::parse(&args); let mut config = Config::default_opts(); + config.minimal_config = MinimalConfig::parse(flags.config.clone()); // Set flags. config.paths = std::mem::take(&mut flags.paths); @@ -918,44 +817,6 @@ impl Config { // Infer the rest of the configuration. - // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, - // running on a completely machine from where it was compiled. - let mut cmd = Command::new("git"); - // NOTE: we cannot support running from outside the repository because the only path we have available - // is set at compile time, which can be wrong if bootstrap was downloaded from source. - // We still support running outside the repository if we find we aren't in a git directory. - cmd.arg("rev-parse").arg("--show-toplevel"); - // Discard stderr because we expect this to fail when building from a tarball. - let output = cmd - .stderr(std::process::Stdio::null()) - .output() - .ok() - .and_then(|output| if output.status.success() { Some(output) } else { None }); - if let Some(output) = output { - let git_root = String::from_utf8(output.stdout).unwrap(); - // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. - let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); - let s = git_root.to_str().unwrap(); - - // Bootstrap is quite bad at handling /? in front of paths - let src = match s.strip_prefix("\\\\?\\") { - Some(p) => PathBuf::from(p), - None => PathBuf::from(git_root), - }; - // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, - // for example, the build directory is inside of another unrelated git directory. - // In that case keep the original `CARGO_MANIFEST_DIR` handling. - // - // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside - // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. - if src.join("src").join("stage0.json").exists() { - config.src = src; - } - } else { - // We're building from a tarball, not git sources. - // We don't support pre-downloaded bootstrap in this case. - } - if cfg!(test) { // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. config.out = Path::new( @@ -1329,17 +1190,19 @@ impl Config { } if config.llvm_from_ci { - let triple = &config.build.triple; + let build_target_selection = config.build; let ci_llvm_bin = config.ci_llvm_root().join("bin"); let build_target = config .target_config .entry(config.build) - .or_insert_with(|| Target::from_triple(&triple)); + .or_insert_with(|| Target::from_triple(&build_target_selection.triple)); check_ci_llvm!(build_target.llvm_config); check_ci_llvm!(build_target.llvm_filecheck); - build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); - build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); + build_target.llvm_config = + Some(ci_llvm_bin.join(exe("llvm-config", build_target_selection))); + build_target.llvm_filecheck = + Some(ci_llvm_bin.join(exe("FileCheck", build_target_selection))); } if let Some(t) = toml.dist { @@ -1448,44 +1311,6 @@ impl Config { config } - pub(crate) fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } - } - - /// A git invocation which runs inside the source directory. - /// - /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. - pub(crate) fn git(&self) -> Command { - let mut git = Command::new("git"); - git.current_dir(&self.src); - git - } - - pub(crate) fn test_args(&self) -> Vec<&str> { - let mut test_args = match self.cmd { - Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => { - test_args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - }; - test_args.extend(self.free_args.iter().map(|s| s.as_str())); - test_args - } - - pub(crate) fn args(&self) -> Vec<&str> { - let mut args = match self.cmd { - Subcommand::Run { ref args, .. } => { - args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - }; - args.extend(self.free_args.iter().map(|s| s.as_str())); - args - } - /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. /// Return the version it would have used for the given commit. pub(crate) fn artifact_version_part(&self, commit: &str) -> String { @@ -1632,12 +1457,6 @@ impl Config { } } - pub fn verbose(&self, msg: &str) { - if self.verbose > 0 { - println!("{}", msg); - } - } - pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { self.target_config.get(&target).map(|t| t.sanitizers).flatten().unwrap_or(self.sanitizers) } @@ -1730,52 +1549,7 @@ impl Config { } }; - // Handle running from a directory other than the top level - let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); - let top_level = top_level.trim_end(); - let compiler = format!("{top_level}/compiler/"); - let library = format!("{top_level}/library/"); - - // Look for a version to compare to based on the current commit. - // Only commits merged by bors will have CI artifacts. - let merge_base = output( - self.git() - .arg("rev-list") - .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) - .args(&["-n1", "--first-parent", "HEAD"]), - ); - let commit = merge_base.trim_end(); - if commit.is_empty() { - println!("error: could not find commit hash for downloading rustc"); - println!("help: maybe your repository history is too shallow?"); - println!("help: consider disabling `download-rustc`"); - println!("help: or fetch enough history to include one upstream commit"); - crate::detail_exit(1); - } - - // Warn if there were changes to the compiler or standard library since the ancestor commit. - let has_changes = !t!(self - .git() - .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) - .status()) - .success(); - if has_changes { - if if_unchanged { - if self.verbose > 0 { - println!( - "warning: saw changes to compiler/ or library/ since {commit}; \ - ignoring `download-rustc`" - ); - } - return None; - } - println!( - "warning: `download-rustc` is enabled, but there are changes to \ - compiler/ or library/" - ); - } - - Some(commit.to_string()) + self.last_modified_commit(&["compiler", "library"], "download-rustc", if_unchanged) } } diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 9cead7adc8c35..37d472e5689fb 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2314,6 +2314,41 @@ impl Step for Bootstrap { } } +/// Tarball intended for being able to run `rustup component add bootstrap-shim`. +/// Not stable, but user-facing. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BootstrapShim { + pub target: TargetSelection, +} + +impl Step for BootstrapShim { + type Output = Option; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // Create this even with only `dist bootstrap` to avoid having to update all CI builders. + run.alias("bootstrap-shim") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BootstrapShim { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + let mut tarball = Tarball::new(builder, "bootstrap-shim", &target.triple); + let bootstrap_outdir = &builder.bootstrap_out; + tarball.add_file( + bootstrap_outdir.join(exe("bootstrap-shim", target)), + tarball.image_dir().join("bin"), + 0o755, + ); + tarball.is_preview(true); + Some(tarball.generate()) + } +} + /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the /// release process to avoid cloning the monorepo and building stuff. /// diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index 3e82a381a1b2d..a4d2f868d0da0 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -11,17 +11,17 @@ use once_cell::sync::OnceCell; use xz2::bufread::XzDecoder; use crate::{ - config::RustfmtMetadata, llvm::detect_llvm_sha, + min_config::RustfmtMetadata, t, - util::{check_run, exe, program_out_of_date, try_run}, - Config, + util::{check_run, exe, output, program_out_of_date, try_run}, + Config, MinimalConfig, }; static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell = OnceCell::new(); /// Generic helpers that are useful anywhere in bootstrap. -impl Config { +impl MinimalConfig { pub fn is_verbose(&self) -> bool { self.verbose > 0 } @@ -329,6 +329,29 @@ impl Config { } return verified; } + + /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. + /// Return the version it would have used for the given commit. + /// + /// NOTE: this currently doesn't support tarballs, use `Config::artifact_version_part` if you need that support. + pub(crate) fn git_artifact_version_part(&self, commit: &str) -> String { + let (channel, version) = { + let mut channel = self.git(); + channel.arg("show").arg(format!("{}:src/ci/channel", commit)); + let channel = output(&mut channel); + let mut version = self.git(); + version.arg("show").arg(format!("{}:src/version", commit)); + let version = output(&mut version); + (channel.trim().to_owned(), version.trim().to_owned()) + }; + + match channel.as_str() { + "stable" => version, + "beta" => channel, + "nightly" => channel, + other => unreachable!("{:?} is not recognized as a valid channel", other), + } + } } enum DownloadSource { @@ -476,7 +499,102 @@ impl Config { /// Download a single component of a CI-built toolchain (not necessarily a published nightly). // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) { - Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc") + self.download_component(DownloadSource::CI, filename, prefix, commit, "ci-rustc") + } + + pub(crate) fn maybe_download_ci_llvm(&self) { + if !self.llvm_from_ci { + return; + } + let llvm_root = self.ci_llvm_root(); + let llvm_stamp = llvm_root.join(".llvm-stamp"); + let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository()); + let key = format!("{}{}", llvm_sha, self.llvm_assertions); + if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { + self.download_ci_llvm(&llvm_sha); + + if self.should_fix_bins_and_dylibs() { + for entry in t!(fs::read_dir(llvm_root.join("bin"))) { + self.fix_bin_or_dylib(&t!(entry).path()); + } + + let llvm_lib = llvm_root.join("lib"); + for entry in t!(fs::read_dir(&llvm_lib)) { + let lib = t!(entry).path(); + if lib.extension().map_or(false, |ext| ext == "so") { + self.fix_bin_or_dylib(&lib); + } + } + } + + // Update the timestamp of llvm-config to force rustc_llvm to be + // rebuilt. This is a hacky workaround for a deficiency in Cargo where + // the rerun-if-changed directive doesn't handle changes very well. + // https://github.com/rust-lang/cargo/issues/10791 + // Cargo only compares the timestamp of the file relative to the last + // time `rustc_llvm` build script ran. However, the timestamps of the + // files in the tarball are in the past, so it doesn't trigger a + // rebuild. + let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); + let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); + t!(filetime::set_file_times(&llvm_config, now, now)); + + t!(fs::write(llvm_stamp, key)); + } + } + + fn download_ci_llvm(&self, llvm_sha: &str) { + let llvm_assertions = self.llvm_assertions; + + let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); + let cache_dst = self.out.join("cache"); + let rustc_cache = cache_dst.join(cache_prefix); + if !rustc_cache.exists() { + t!(fs::create_dir_all(&rustc_cache)); + } + let base = if llvm_assertions { + &self.stage0_metadata.config.artifacts_with_llvm_assertions_server + } else { + &self.stage0_metadata.config.artifacts_server + }; + let version = self.artifact_version_part(llvm_sha); + let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); + let tarball = rustc_cache.join(&filename); + if !tarball.exists() { + let help_on_error = "error: failed to download llvm from ci + + help: old builds get deleted after a certain time + help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: + + [llvm] + download-ci-llvm = false + "; + self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error); + } + let llvm_root = self.ci_llvm_root(); + self.unpack(&tarball, &llvm_root, "rust-dev"); + } +} + +impl MinimalConfig { + pub fn download_bootstrap(&self, commit: &str) -> PathBuf { + self.verbose(&format!("downloading bootstrap from CI (commit {commit})")); + let host = self.build.triple; + let bin_root = self.out.join(host).join("bootstrap"); + let stamp = bin_root.join(".bootstrap-stamp"); + let bootstrap_bin = bin_root.join("bin").join("bootstrap"); + + if !bootstrap_bin.exists() || program_out_of_date(&stamp, commit) { + let version = self.git_artifact_version_part(commit); + let filename = format!("bootstrap-{version}-{host}.tar.xz"); + self.download_component(DownloadSource::CI, filename, "bootstrap", commit, ""); + if self.should_fix_bins_and_dylibs() { + self.fix_bin_or_dylib(&bootstrap_bin); + } + t!(fs::write(stamp, commit)); + } + + bootstrap_bin } fn download_component( @@ -560,78 +678,4 @@ download-rustc = false self.unpack(&tarball, &bin_root, prefix); } - - pub(crate) fn maybe_download_ci_llvm(&self) { - if !self.llvm_from_ci { - return; - } - let llvm_root = self.ci_llvm_root(); - let llvm_stamp = llvm_root.join(".llvm-stamp"); - let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository()); - let key = format!("{}{}", llvm_sha, self.llvm_assertions); - if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { - self.download_ci_llvm(&llvm_sha); - if self.should_fix_bins_and_dylibs() { - for entry in t!(fs::read_dir(llvm_root.join("bin"))) { - self.fix_bin_or_dylib(&t!(entry).path()); - } - } - - // Update the timestamp of llvm-config to force rustc_llvm to be - // rebuilt. This is a hacky workaround for a deficiency in Cargo where - // the rerun-if-changed directive doesn't handle changes very well. - // https://github.com/rust-lang/cargo/issues/10791 - // Cargo only compares the timestamp of the file relative to the last - // time `rustc_llvm` build script ran. However, the timestamps of the - // files in the tarball are in the past, so it doesn't trigger a - // rebuild. - let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); - let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); - t!(filetime::set_file_times(&llvm_config, now, now)); - - if self.should_fix_bins_and_dylibs() { - let llvm_lib = llvm_root.join("lib"); - for entry in t!(fs::read_dir(&llvm_lib)) { - let lib = t!(entry).path(); - if lib.extension().map_or(false, |ext| ext == "so") { - self.fix_bin_or_dylib(&lib); - } - } - } - - t!(fs::write(llvm_stamp, key)); - } - } - - fn download_ci_llvm(&self, llvm_sha: &str) { - let llvm_assertions = self.llvm_assertions; - - let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); - let cache_dst = self.out.join("cache"); - let rustc_cache = cache_dst.join(cache_prefix); - if !rustc_cache.exists() { - t!(fs::create_dir_all(&rustc_cache)); - } - let base = if llvm_assertions { - &self.stage0_metadata.config.artifacts_with_llvm_assertions_server - } else { - &self.stage0_metadata.config.artifacts_server - }; - let version = self.artifact_version_part(llvm_sha); - let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); - let tarball = rustc_cache.join(&filename); - if !tarball.exists() { - let help_on_error = "error: failed to download llvm from ci - - help: old builds get deleted after a certain time - help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: - - [llvm] - download-ci-llvm = false - "; - self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error); - } - let llvm_root = self.ci_llvm_root(); - self.unpack(&tarball, &llvm_root, "rust-dev"); - } } diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 6e0c0e01af863..c7d93555448fd 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -8,7 +8,8 @@ use std::path::{Path, PathBuf}; use clap::{CommandFactory, Parser, ValueEnum}; use crate::builder::{Builder, Kind}; -use crate::config::{target_selection_list, Config, TargetSelectionList}; +use crate::config::Config; +use crate::min_config::{target_selection_list, TargetSelectionList}; use crate::setup::Profile; use crate::{Build, DocTests}; diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 3756976dee062..597c257566b0d 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -55,6 +55,7 @@ mod format; mod install; mod llvm; mod metadata; +mod min_config; mod render_tests; mod run; mod sanity; @@ -90,6 +91,7 @@ pub use crate::builder::PathSet; use crate::cache::{Interned, INTERNER}; pub use crate::config::Config; pub use crate::flags::Subcommand; +pub use crate::min_config::MinimalConfig; use termcolor::{ColorChoice, StandardStream, WriteColor}; const LLVM_TOOLS: &[&str] = &[ diff --git a/src/bootstrap/min_config.rs b/src/bootstrap/min_config.rs new file mode 100644 index 0000000000000..92ffa4382de2d --- /dev/null +++ b/src/bootstrap/min_config.rs @@ -0,0 +1,393 @@ +use core::fmt; +use std::{ + collections::HashMap, + env, fs, + path::{Path, PathBuf}, + process::Command, +}; + +use serde::Deserialize; +use serde_derive::Deserialize; + +use crate::{ + cache::{Interned, INTERNER}, + t, + util::output, +}; + +/// The bare minimum config, suitable for `bootstrap-shim`, but sharing code with the main `bootstrap` binary. +#[derive(Default, Clone)] +pub struct MinimalConfig { + // Needed so we know where to store the unpacked bootstrap binary. + pub build: TargetSelection, + // Needed so we know where to load `src/stage0.json` + pub src: PathBuf, + // Needed so we know where to store the cache. + pub out: PathBuf, + pub patch_binaries_for_nix: bool, + // Needed to know which commit to download. + pub stage0_metadata: Stage0Metadata, + + // This isn't currently used, but will eventually let people configure whether to download or build bootstrap. + pub config: Option, + // Not currently used in the shim. + pub verbose: usize, + // Not currently used in the shim. + pub dry_run: DryRun, +} + +#[derive(Default, Deserialize, Clone)] +pub struct Stage0Metadata { + pub compiler: CompilerMetadata, + pub config: Stage0Config, + pub checksums_sha256: HashMap, + pub rustfmt: Option, +} +#[derive(Clone, Default, Deserialize)] +pub struct CompilerMetadata { + pub date: String, + pub version: String, +} +#[derive(Default, Deserialize, Clone)] +pub struct Stage0Config { + pub dist_server: String, + pub artifacts_server: String, + pub artifacts_with_llvm_assertions_server: String, + pub git_merge_commit_email: String, + pub nightly_branch: String, +} + +#[derive(Default, Deserialize, Clone)] +pub struct RustfmtMetadata { + pub date: String, + pub version: String, +} + +impl MinimalConfig { + fn default_opts() -> Self { + let dry_run = DryRun::default(); + let config = None; + let verbose = 0; + let patch_binaries_for_nix = false; + let stage0_metadata = Stage0Metadata::default(); + + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + let src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); + let out = PathBuf::from("build"); + + // set by build.rs + let build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); + + MinimalConfig { + build, + src, + out, + config, + dry_run, + verbose, + patch_binaries_for_nix, + stage0_metadata, + } + } + + pub fn parse(config_flag: Option) -> MinimalConfig { + let mut config = Self::default_opts(); + + if let Some(src) = src() { + config.src = src; + } + + if cfg!(test) { + // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. + config.out = Path::new( + &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), + ) + .parent() + .unwrap() + .to_path_buf(); + } + + let toml = if let Some(toml_path) = Self::config_path(config.src.clone(), config_flag) { + config.config = Some(toml_path.clone()); + get_toml(&toml_path) + } else { + config.config = None; + TomlConfig::default() + }; + if let Some(build) = toml.build.unwrap_or_default().build { + config.build = TargetSelection::from_user(&build); + } + + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + if !config.out.is_absolute() { + // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + config.out = crate::util::absolute(&config.out); + } + + if config.dry_run() { + let dir = config.out.join("tmp-dry-run"); + t!(fs::create_dir_all(&dir)); + config.out = dir; + } + + let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); + config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); + + config + } + + /// Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. + /// + /// Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + /// but not if `config.toml` hasn't been created. + fn config_path(src: PathBuf, config_flag: Option) -> Option { + let toml_path = + config_flag.or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); + let using_default_path = toml_path.is_none(); + let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); + if using_default_path && !toml_path.exists() { + toml_path = src.join(toml_path); + } + + if !using_default_path || toml_path.exists() { Some(toml_path) } else { None } + } +} + +impl MinimalConfig { + pub fn verbose(&self, msg: &str) { + if self.verbose > 0 { + println!("{}", msg); + } + } + + pub(crate) fn dry_run(&self) -> bool { + match self.dry_run { + DryRun::Disabled => false, + DryRun::SelfCheck | DryRun::UserSelected => true, + } + } + + /// A git invocation which runs inside the source directory. + /// + /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. + pub(crate) fn git(&self) -> Command { + let mut git = Command::new("git"); + git.current_dir(&self.src); + git + } + + /// Returns the last commit in which any of `modified_paths` were changed, + /// or `None` if there are untracked changes in the working directory and `if_unchanged` is true. + pub fn last_modified_commit( + &self, + modified_paths: &[&str], + option_name: &str, + if_unchanged: bool, + ) -> Option { + // Handle running from a directory other than the top level + let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output( + self.git() + .arg("rev-list") + .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("error: could not find commit hash for downloading components from CI"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `{option_name}`"); + println!("help: or fetch enough history to include one upstream commit"); + crate::detail_exit(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let mut git = self.git(); + git.args(&["diff-index", "--quiet", &commit, "--"]); + + for path in modified_paths { + git.arg(format!("{top_level}/{path}")); + } + + let has_changes = !t!(git.status()).success(); + if has_changes { + if if_unchanged { + if self.verbose > 0 { + println!( + "warning: saw changes to one of {modified_paths:?} since {commit}; \ + ignoring `{option_name}`" + ); + } + return None; + } + println!( + "warning: `{option_name}` is enabled, but there are changes to one of {modified_paths:?}" + ); + } + + Some(commit.to_string()) + } +} + +#[cfg(test)] +pub(crate) fn get_toml + Default>(_file: &Path) -> T { + T::default() +} +#[cfg(not(test))] +pub(crate) fn get_toml + Default>(file: &Path) -> T { + let contents = + t!(fs::read_to_string(file), format!("config file {} not found", file.display())); + // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of + // TomlConfig and sub types to be monomorphized 5x by toml. + match toml::from_str(&contents).and_then(|table: toml::Value| T::deserialize(table)) { + Ok(table) => table, + Err(err) => { + eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err); + crate::detail_exit(2); + } + } +} + +fn src() -> Option { + // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, + // running on a completely machine from where it was compiled. + let mut cmd = Command::new("git"); + // NOTE: we cannot support running from outside the repository because the only path we have available + // is set at compile time, which can be wrong if bootstrap was downloaded from source. + // We still support running outside the repository if we find we aren't in a git directory. + cmd.arg("rev-parse").arg("--show-toplevel"); + // Discard stderr because we expect this to fail when building from a tarball. + let output = cmd + .stderr(std::process::Stdio::null()) + .output() + .ok() + .and_then(|output| if output.status.success() { Some(output) } else { None }); + if let Some(output) = output { + let git_root = String::from_utf8(output.stdout).unwrap(); + // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. + let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); + let s = git_root.to_str().unwrap(); + + // Bootstrap is quite bad at handling /? in front of paths + let src = match s.strip_prefix("\\\\?\\") { + Some(p) => PathBuf::from(p), + None => PathBuf::from(git_root), + }; + // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, + // for example, the build directory is inside of another unrelated git directory. + // In that case keep the original `CARGO_MANIFEST_DIR` handling. + // + // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside + // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. + if src.join("src").join("stage0.json").exists() { Some(src) } else { None } + } else { + // We're building from a tarball, not git sources. + // We don't support pre-downloaded bootstrap in this case. + None + } +} + +#[derive(Clone, Default)] +pub enum DryRun { + /// This isn't a dry run. + #[default] + Disabled, + /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. + SelfCheck, + /// This is a dry run enabled by the `--dry-run` flag. + UserSelected, +} + +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TargetSelection { + pub triple: Interned, + file: Option>, +} + +/// Newtype over `Vec` so we can implement custom parsing logic +#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct TargetSelectionList(pub(crate) Vec); + +pub fn target_selection_list(s: &str) -> Result { + Ok(TargetSelectionList( + s.split(",").filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), + )) +} + +impl TargetSelection { + pub fn from_user(selection: &str) -> Self { + let path = Path::new(selection); + + let (triple, file) = if path.exists() { + let triple = path + .file_stem() + .expect("Target specification file has no file stem") + .to_str() + .expect("Target specification file stem is not UTF-8"); + + (triple, Some(selection)) + } else { + (selection, None) + }; + + let triple = INTERNER.intern_str(triple); + let file = file.map(|f| INTERNER.intern_str(f)); + + Self { triple, file } + } + + pub fn rustc_target_arg(&self) -> &str { + self.file.as_ref().unwrap_or(&self.triple) + } + + pub fn contains(&self, needle: &str) -> bool { + self.triple.contains(needle) + } + + pub fn starts_with(&self, needle: &str) -> bool { + self.triple.starts_with(needle) + } + + pub fn ends_with(&self, needle: &str) -> bool { + self.triple.ends_with(needle) + } +} + +impl fmt::Display for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.triple)?; + if let Some(file) = self.file { + write!(f, "({})", file)?; + } + Ok(()) + } +} + +impl fmt::Debug for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self) + } +} + +impl PartialEq<&str> for TargetSelection { + fn eq(&self, other: &&str) -> bool { + self.triple == *other + } +} + +#[derive(Deserialize, Default)] +struct TomlConfig { + build: Option, +} + +/// TOML representation of various global build decisions. +#[derive(Deserialize, Default)] +struct Build { + build: Option, +} diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 04fdb15f5acb2..4983a41fc3685 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -84,7 +84,7 @@ ENV RUST_CONFIGURE_ARGS \ ENV SCRIPT python3 ../src/ci/stage-build.py python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ --include-default-paths \ - build-manifest bootstrap + build-manifest bootstrap bootstrap-shim ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang # This is the only builder which will create source tarballs diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index f81e740936ba5..ee993eb02f22b 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -489,7 +489,7 @@ jobs: - name: dist-x86_64-apple env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -502,7 +502,7 @@ jobs: - name: dist-apple-various env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -514,7 +514,7 @@ jobs: - name: dist-x86_64-apple-alt env: - SCRIPT: ./x.py dist bootstrap --include-default-paths + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -545,7 +545,7 @@ jobs: # This target only needs to support 11.0 and up as nothing else supports the hardware - name: dist-aarch64-apple env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --stage 2 + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths --stage 2 RUST_CONFIGURE_ARGS: >- --build=x86_64-apple-darwin --host=aarch64-apple-darwin @@ -684,7 +684,7 @@ jobs: --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths + SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c @@ -696,7 +696,7 @@ jobs: --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c @@ -707,7 +707,7 @@ jobs: --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 # Hack around this SDK version, because it doesn't work with clang. # See https://github.com/rust-lang/rust/issues/88796 @@ -723,14 +723,14 @@ jobs: # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - name: dist-x86_64-mingw env: - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-gnu --enable-full-tools @@ -745,7 +745,7 @@ jobs: - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths <<: *job-windows-8c try: diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 21dad9eb74aa9..ec0dbc3c3badc 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -183,7 +183,8 @@ static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin" static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"]; -static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs]; +static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = + &[PkgType::Miri, PkgType::JsonDocs, PkgType::BootstrapShim]; macro_rules! t { ($e:expr) => { @@ -414,7 +415,8 @@ impl Builder { | PkgType::Rustfmt | PkgType::LlvmTools | PkgType::RustAnalysis - | PkgType::JsonDocs => { + | PkgType::JsonDocs + | PkgType::BootstrapShim => { extensions.push(host_component(pkg)); } PkgType::RustcDev | PkgType::RustcDocs => { diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 7a4c15d01eadc..9fd84b011c523 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -57,6 +57,7 @@ pkg_type! { LlvmTools = "llvm-tools"; preview = true, Miri = "miri"; preview = true, JsonDocs = "rust-docs-json"; preview = true, + BootstrapShim = "bootstrap-shim"; preview = true, } impl PkgType { @@ -92,6 +93,7 @@ impl PkgType { PkgType::ReproducibleArtifacts => true, PkgType::RustMingw => true, PkgType::RustAnalysis => true, + PkgType::BootstrapShim => true, } } @@ -115,6 +117,7 @@ impl PkgType { RustAnalyzer => HOSTS, Clippy => HOSTS, Miri => HOSTS, + BootstrapShim => HOSTS, Rustfmt => HOSTS, RustAnalysis => TARGETS, LlvmTools => TARGETS, diff --git a/src/tools/x/src/main.rs b/src/tools/x/src/main.rs index 5da8a2888ec83..344cb3e48b76f 100644 --- a/src/tools/x/src/main.rs +++ b/src/tools/x/src/main.rs @@ -98,7 +98,7 @@ fn handle_result(result: io::Result, cmd: Command) { } } -fn main() { +pub fn main() { match env::args().skip(1).next().as_deref() { Some("--wrapper-version") => { let version = env!("CARGO_PKG_VERSION"); From 8ae63b1309a34a56c7f9260f91e90e7b6b001f64 Mon Sep 17 00:00:00 2001 From: ozkanonur Date: Mon, 30 Jan 2023 16:51:14 +0300 Subject: [PATCH 2/5] Unify `MinimalConfig::parse` and `Config::parse` Signed-off-by: ozkanonur --- .github/workflows/ci.yml | 20 +++---- src/bootstrap/config.rs | 41 +++----------- src/bootstrap/min_config.rs | 107 ++++++++++++++++++++++-------------- 3 files changed, 83 insertions(+), 85 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d8980fcd1a0d..1f64e71d1a480 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -311,7 +311,7 @@ jobs: os: ubuntu-20.04-8core-32gb - name: dist-x86_64-apple env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin" RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -323,7 +323,7 @@ jobs: os: macos-latest - name: dist-apple-various env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim" RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -334,7 +334,7 @@ jobs: os: macos-latest - name: dist-x86_64-apple-alt env: - SCRIPT: "./x.py dist bootstrap --include-default-paths" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths" RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -367,7 +367,7 @@ jobs: os: macos-latest - name: dist-aarch64-apple env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --stage 2" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --stage 2" RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --disable-docs --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 SELECT_XCODE: /Applications/Xcode_13.4.1.app @@ -442,19 +442,19 @@ jobs: - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths + SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-2019-8core-32gb - name: dist-i686-msvc env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-2019-8core-32gb - name: dist-aarch64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 WINDOWS_SDK_20348_HACK: 1 os: windows-2019-8core-32gb @@ -462,13 +462,13 @@ jobs: env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler" NO_DOWNLOAD_CI_LLVM: 1 - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 os: windows-2019-8core-32gb - name: dist-x86_64-mingw env: - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler" NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 @@ -477,7 +477,7 @@ jobs: - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths os: windows-2019-8core-32gb timeout-minutes: 600 runs-on: "${{ matrix.os }}" diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 01ec697fb353f..ed9ac33dda328 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -22,7 +22,9 @@ use crate::cc_detect::{ndk_compiler, Language}; use crate::channel::{self, GitInfo}; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags, Warnings}; -use crate::min_config::get_toml; +use crate::min_config::{ + deserialize_stage0_metadata, get_toml, set_and_return_toml_config, set_config_output_dir, +}; use crate::util::{exe, output, t}; use crate::MinimalConfig; use once_cell::sync::OnceCell; @@ -817,40 +819,11 @@ impl Config { // Infer the rest of the configuration. - if cfg!(test) { - // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. - config.out = Path::new( - &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), - ) - .parent() - .unwrap() - .to_path_buf(); - } - - let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); - - config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); + set_config_output_dir(&mut config.out); + config.stage0_metadata = deserialize_stage0_metadata(&config.src); - // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. - let toml_path = flags - .config - .clone() - .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); - let using_default_path = toml_path.is_none(); - let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); - if using_default_path && !toml_path.exists() { - toml_path = config.src.join(toml_path); - } - - // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, - // but not if `config.toml` hasn't been created. - let mut toml = if !using_default_path || toml_path.exists() { - config.config = Some(toml_path.clone()); - get_toml(&toml_path) - } else { - config.config = None; - TomlConfig::default() - }; + let mut toml: TomlConfig = + set_and_return_toml_config(config.src.clone(), flags.config, &mut config.config); if let Some(include) = &toml.profile { let mut include_path = config.src.clone(); diff --git a/src/bootstrap/min_config.rs b/src/bootstrap/min_config.rs index 92ffa4382de2d..b0088794b7f26 100644 --- a/src/bootstrap/min_config.rs +++ b/src/bootstrap/min_config.rs @@ -98,61 +98,31 @@ impl MinimalConfig { config.src = src; } - if cfg!(test) { - // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. - config.out = Path::new( - &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), - ) - .parent() - .unwrap() - .to_path_buf(); - } + set_config_output_dir(&mut config.out); + + let toml: TomlConfig = + set_and_return_toml_config(config.src.clone(), config_flag, &mut config.config); - let toml = if let Some(toml_path) = Self::config_path(config.src.clone(), config_flag) { - config.config = Some(toml_path.clone()); - get_toml(&toml_path) - } else { - config.config = None; - TomlConfig::default() - }; if let Some(build) = toml.build.unwrap_or_default().build { config.build = TargetSelection::from_user(&build); } - // NOTE: Bootstrap spawns various commands with different working directories. - // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. - if !config.out.is_absolute() { - // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. - config.out = crate::util::absolute(&config.out); - } - if config.dry_run() { let dir = config.out.join("tmp-dry-run"); t!(fs::create_dir_all(&dir)); config.out = dir; } + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + else if !config.out.is_absolute() { + // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + config.out = crate::util::absolute(&config.out); + } - let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); - config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); + config.stage0_metadata = deserialize_stage0_metadata(&config.src); config } - - /// Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. - /// - /// Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, - /// but not if `config.toml` hasn't been created. - fn config_path(src: PathBuf, config_flag: Option) -> Option { - let toml_path = - config_flag.or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); - let using_default_path = toml_path.is_none(); - let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); - if using_default_path && !toml_path.exists() { - toml_path = src.join(toml_path); - } - - if !using_default_path || toml_path.exists() { Some(toml_path) } else { None } - } } impl MinimalConfig { @@ -236,10 +206,12 @@ impl MinimalConfig { } #[cfg(test)] +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` pub(crate) fn get_toml + Default>(_file: &Path) -> T { T::default() } #[cfg(not(test))] +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` pub(crate) fn get_toml + Default>(file: &Path) -> T { let contents = t!(fs::read_to_string(file), format!("config file {} not found", file.display())); @@ -254,6 +226,59 @@ pub(crate) fn get_toml + Default>(file: &Path) -> T { } } +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +/// +/// Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. +#[allow(unused_variables)] +pub(crate) fn set_config_output_dir(output_path: &mut PathBuf) { + #[cfg(test)] + { + *output_path = Path::new( + &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), + ) + .parent() + .unwrap() + .to_path_buf(); + } +} + +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +pub(crate) fn set_and_return_toml_config + Default>( + src: PathBuf, + config_flag: Option, + cfg_path: &mut Option, +) -> T { + /// Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. + /// + /// Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + /// but not if `config.toml` hasn't been created. + fn config_path(src: &PathBuf, config_flag: Option) -> Option { + let toml_path = + config_flag.or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); + let using_default_path = toml_path.is_none(); + let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); + if using_default_path && !toml_path.exists() { + toml_path = src.join(toml_path); + } + + if !using_default_path || toml_path.exists() { Some(toml_path) } else { None } + } + + if let Some(toml_path) = config_path(&src, config_flag) { + *cfg_path = Some(toml_path.clone()); + get_toml(&toml_path) + } else { + *cfg_path = None; + T::default() + } +} + +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +pub(crate) fn deserialize_stage0_metadata(stage0_metadata_path: &PathBuf) -> Stage0Metadata { + let stage0_json = t!(std::fs::read(stage0_metadata_path.join("src").join("stage0.json"))); + t!(serde_json::from_slice::(&stage0_json)) +} + fn src() -> Option { // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, // running on a completely machine from where it was compiled. From 8a6c18fbe9d641a4bca45c9398648de6f824b219 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 1 Feb 2023 22:14:08 -0600 Subject: [PATCH 3/5] Also re-download bootstrap if `build_helper` changes --- src/bootstrap/bin/bootstrap-shim.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/bin/bootstrap-shim.rs b/src/bootstrap/bin/bootstrap-shim.rs index d0d044d2373a8..91990bd795e14 100644 --- a/src/bootstrap/bin/bootstrap-shim.rs +++ b/src/bootstrap/bin/bootstrap-shim.rs @@ -25,5 +25,9 @@ fn main() { } fn last_modified_bootstrap_commit(config: &MinimalConfig) -> Option { - config.last_modified_commit(&["src/bootstrap"], "download-bootstrap", true) + config.last_modified_commit( + &["src/bootstrap", "src/tools/build_helper"], + "download-bootstrap", + true, + ) } From a8f85aed2a06e6153e94f9a8380d0031c77b5604 Mon Sep 17 00:00:00 2001 From: ozkanonur Date: Wed, 8 Feb 2023 20:53:25 +0300 Subject: [PATCH 4/5] move `min_config` related options from `config` module Signed-off-by: ozkanonur --- src/bootstrap/bin/bootstrap-shim.rs | 8 ++-- src/bootstrap/bin/main.rs | 2 +- src/bootstrap/builder/tests.rs | 4 +- src/bootstrap/config.rs | 61 +++++++++++------------------ src/bootstrap/config/tests.rs | 6 +-- src/bootstrap/flags.rs | 2 +- src/bootstrap/lib.rs | 1 + src/bootstrap/min_config.rs | 39 +++++++++++++----- 8 files changed, 63 insertions(+), 60 deletions(-) diff --git a/src/bootstrap/bin/bootstrap-shim.rs b/src/bootstrap/bin/bootstrap-shim.rs index 91990bd795e14..d9a92ecda03fe 100644 --- a/src/bootstrap/bin/bootstrap-shim.rs +++ b/src/bootstrap/bin/bootstrap-shim.rs @@ -1,19 +1,17 @@ use std::{env, process::Command}; -use bootstrap::{t, MinimalConfig}; +use bootstrap::{Flags, MinimalConfig}; #[path = "../../../src/tools/x/src/main.rs"] mod run_python; fn main() { let args = env::args().skip(1).collect::>(); - let mut opts = getopts::Options::new(); - opts.optopt("", "config", "TOML configuration file for build", "FILE"); - let matches = t!(opts.parse(args)); + let flags = Flags::parse(&args); // If there are no untracked changes to bootstrap, download it from CI. // Otherwise, build it from source. Use python to build to avoid duplicating the code between python and rust. - let config = MinimalConfig::parse(t!(matches.opt_get("config"))); + let config = MinimalConfig::parse(&flags, None); let bootstrap_bin = if let Some(commit) = last_modified_bootstrap_commit(&config) { config.download_bootstrap(&commit) } else { diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/bin/main.rs index a80379e85c193..4d8b4f2f359e2 100644 --- a/src/bootstrap/bin/main.rs +++ b/src/bootstrap/bin/main.rs @@ -13,7 +13,7 @@ use bootstrap::{Build, Config, Subcommand, VERSION}; fn main() { let args = env::args().skip(1).collect::>(); - let config = Config::parse(&args); + let config = Config::parse(&args, None); #[cfg(all(any(unix, windows), not(target_os = "solaris")))] let mut build_lock; diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index c32fe59bbf069..08ea0a1e57c88 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -7,7 +7,7 @@ fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { } fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config { - let mut config = Config::parse(cmd); + let mut config = Config::parse(cmd, None); // don't save toolstates config.save_toolstates = None; config.dry_run = DryRun::SelfCheck; @@ -18,7 +18,7 @@ fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config let submodule_build = Build::new(Config { // don't include LLVM, so CI doesn't require ninja/cmake to be installed rust_codegen_backends: vec![], - ..Config::parse(&["check".to_owned()]) + ..Config::parse(&["check".to_owned()], None) }); submodule_build.update_submodule(Path::new("src/doc/book")); submodule_build.update_submodule(Path::new("src/tools/rust-analyzer")); diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index ed9ac33dda328..25760fb86ab0b 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -7,9 +7,7 @@ mod tests; use std::cell::{Cell, RefCell}; -use std::cmp; use std::collections::{HashMap, HashSet}; -use std::env; use std::fs; use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; @@ -22,9 +20,7 @@ use crate::cc_detect::{ndk_compiler, Language}; use crate::channel::{self, GitInfo}; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags, Warnings}; -use crate::min_config::{ - deserialize_stage0_metadata, get_toml, set_and_return_toml_config, set_config_output_dir, -}; +use crate::min_config::{get_toml, set_cfg_path_and_return_toml_cfg}; use crate::util::{exe, output, t}; use crate::MinimalConfig; use once_cell::sync::OnceCell; @@ -391,6 +387,7 @@ impl Target { target } } + /// Structure of the `config.toml` file that configuration is read from. /// /// This structure uses `Decodable` to automatically decode a TOML configuration @@ -398,7 +395,7 @@ impl Target { /// `Config` structure. #[derive(Deserialize, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] -struct TomlConfig { +pub struct TomlConfig { changelog_seen: Option, build: Option, install: Option, @@ -443,8 +440,8 @@ macro_rules! define_config { $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* }) => { $(#[$attr])* - struct $name { - $($field: Option<$field_ty>,)* + pub struct $name { + $(pub(crate) $field: Option<$field_ty>,)* } impl Merge for $name { @@ -526,7 +523,7 @@ macro_rules! define_config { define_config! { /// TOML representation of various global build decisions. - #[derive(Default)] + #[derive(Clone, Default)] struct Build { build: Option = "build", host: Option> = "host", @@ -632,7 +629,7 @@ define_config! { #[derive(Debug, Deserialize)] #[serde(untagged)] -enum StringOrBool { +pub(crate) enum StringOrBool { String(String), Bool(bool), } @@ -752,14 +749,6 @@ impl Config { config.stdout_is_tty = std::io::stdout().is_terminal(); config.stderr_is_tty = std::io::stderr().is_terminal(); - // set by build.rs - config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); - - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - // Undo `src/bootstrap` - config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); - config.out = PathBuf::from("build"); - config } @@ -785,10 +774,19 @@ impl Config { args } - pub fn parse(args: &[String]) -> Config { + pub fn parse(args: &[String], custom_toml_config: Option) -> Config { let mut flags = Flags::parse(&args); let mut config = Config::default_opts(); - config.minimal_config = MinimalConfig::parse(flags.config.clone()); + + let mut toml: TomlConfig = custom_toml_config.unwrap_or_else(|| { + set_cfg_path_and_return_toml_cfg( + config.src.clone(), + flags.config.clone(), + &mut config.config, + ) + }); + + config.minimal_config = MinimalConfig::parse(&flags, toml.build.clone()); // Set flags. config.paths = std::mem::take(&mut flags.paths); @@ -800,7 +798,6 @@ impl Config { config.jobs = Some(threads_from_config(flags.jobs as u32)); config.cmd = flags.cmd; config.incremental = flags.incremental; - config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; config.keep_stage = flags.keep_stage; config.keep_stage_std = flags.keep_stage_std; config.color = flags.color; @@ -817,13 +814,10 @@ impl Config { crate::detail_exit(1); } - // Infer the rest of the configuration. - - set_config_output_dir(&mut config.out); - config.stage0_metadata = deserialize_stage0_metadata(&config.src); - - let mut toml: TomlConfig = - set_and_return_toml_config(config.src.clone(), flags.config, &mut config.config); + let build = toml.build.clone().unwrap_or_default(); + if let Some(file_build) = build.build.as_ref() { + config.build = TargetSelection::from_user(file_build); + }; if let Some(include) = &toml.profile { let mut include_path = config.src.clone(); @@ -837,11 +831,6 @@ impl Config { config.changelog_seen = toml.changelog_seen; - let build = toml.build.unwrap_or_default(); - if let Some(file_build) = build.build { - config.build = TargetSelection::from_user(&file_build); - }; - set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from))); // NOTE: Bootstrap spawns various commands with different working directories. // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. @@ -902,7 +891,6 @@ impl Config { set(&mut config.full_bootstrap, build.full_bootstrap); set(&mut config.extended, build.extended); config.tools = build.tools; - set(&mut config.verbose, build.verbose); set(&mut config.sanitizers, build.sanitizers); set(&mut config.profiler, build.profiler); set(&mut config.cargo_native_static, build.cargo_native_static); @@ -910,9 +898,6 @@ impl Config { set(&mut config.local_rebuild, build.local_rebuild); set(&mut config.print_step_timings, build.print_step_timings); set(&mut config.print_step_rusage, build.print_step_rusage); - set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix); - - config.verbose = cmp::max(config.verbose, flags.verbose as usize); if let Some(install) = toml.install { config.prefix = install.prefix.map(PathBuf::from); @@ -1526,7 +1511,7 @@ impl Config { } } -fn set(field: &mut T, val: Option) { +pub(crate) fn set(field: &mut T, val: Option) { if let Some(v) = val { *field = v; } diff --git a/src/bootstrap/config/tests.rs b/src/bootstrap/config/tests.rs index d913ca295e29d..3815d6517be42 100644 --- a/src/bootstrap/config/tests.rs +++ b/src/bootstrap/config/tests.rs @@ -2,12 +2,12 @@ use super::{Config, Flags, TomlConfig}; use clap::CommandFactory; use std::{env, path::Path}; -fn toml(config: &str) -> impl '_ + Fn(&Path) -> TomlConfig { - |&_| toml::from_str(config).unwrap() +fn toml(config: &str) -> TomlConfig { + toml::from_str(config).unwrap() } fn parse(config: &str) -> Config { - Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], toml(config)) + Config::parse(&["check".to_owned(), "--config=/does/not/exist".to_owned()], Some(toml(config))) } #[test] diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index c7d93555448fd..35e7b9f5ccfa4 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -183,7 +183,7 @@ impl Flags { HelpVerboseOnly::try_parse_from(it.clone()) { println!("note: updating submodules before printing available paths"); - let config = Config::parse(&[String::from("build")]); + let config = Config::parse(&[String::from("build")], None); let build = Build::new(config); let paths = Builder::get_help(&build, subcommand); if let Some(s) = paths { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 597c257566b0d..dda09852439de 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -90,6 +90,7 @@ mod job { pub use crate::builder::PathSet; use crate::cache::{Interned, INTERNER}; pub use crate::config::Config; +pub use crate::flags::Flags; pub use crate::flags::Subcommand; pub use crate::min_config::MinimalConfig; use termcolor::{ColorChoice, StandardStream, WriteColor}; diff --git a/src/bootstrap/min_config.rs b/src/bootstrap/min_config.rs index b0088794b7f26..9282cf609c928 100644 --- a/src/bootstrap/min_config.rs +++ b/src/bootstrap/min_config.rs @@ -1,5 +1,6 @@ use core::fmt; use std::{ + cmp, collections::HashMap, env, fs, path::{Path, PathBuf}, @@ -11,6 +12,8 @@ use serde_derive::Deserialize; use crate::{ cache::{Interned, INTERNER}, + config::{self, set}, + flags::Flags, t, util::output, }; @@ -91,22 +94,40 @@ impl MinimalConfig { } } - pub fn parse(config_flag: Option) -> MinimalConfig { + fn set_shared_fields_from_parent(&mut self, parent_build_config: config::Build) { + set(&mut self.verbose, parent_build_config.verbose); + set(&mut self.patch_binaries_for_nix, parent_build_config.patch_binaries_for_nix); + } + + pub fn parse(flags: &Flags, parent_build_config: Option) -> MinimalConfig { let mut config = Self::default_opts(); + if let Some(parent_build_config) = parent_build_config { + config.set_shared_fields_from_parent(parent_build_config); + }; + if let Some(src) = src() { config.src = src; } set_config_output_dir(&mut config.out); - let toml: TomlConfig = - set_and_return_toml_config(config.src.clone(), config_flag, &mut config.config); + config.stage0_metadata = deserialize_stage0_metadata(&config.src); + config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; + + let toml: TomlConfig = set_cfg_path_and_return_toml_cfg( + config.src.clone(), + flags.config.clone(), + &mut config.config, + ); - if let Some(build) = toml.build.unwrap_or_default().build { - config.build = TargetSelection::from_user(&build); + let build = toml.build.unwrap_or_default(); + if let Some(file_build) = build.build { + config.build = TargetSelection::from_user(&file_build); } + config.verbose = cmp::max(config.verbose, flags.verbose as usize); + if config.dry_run() { let dir = config.out.join("tmp-dry-run"); t!(fs::create_dir_all(&dir)); @@ -229,10 +250,8 @@ pub(crate) fn get_toml + Default>(file: &Path) -> T { /// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` /// /// Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. -#[allow(unused_variables)] -pub(crate) fn set_config_output_dir(output_path: &mut PathBuf) { - #[cfg(test)] - { +fn set_config_output_dir(output_path: &mut PathBuf) { + if cfg!(test) { *output_path = Path::new( &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), ) @@ -243,7 +262,7 @@ pub(crate) fn set_config_output_dir(output_path: &mut PathBuf) { } /// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` -pub(crate) fn set_and_return_toml_config + Default>( +pub(crate) fn set_cfg_path_and_return_toml_cfg + Default>( src: PathBuf, config_flag: Option, cfg_path: &mut Option, From 8ad89068e2558940b4184d46763ff9c6b325883b Mon Sep 17 00:00:00 2001 From: ozkanonur Date: Tue, 16 May 2023 14:52:16 +0300 Subject: [PATCH 5/5] refactor `src/tools/x` Signed-off-by: ozkanonur --- src/bootstrap/bin/bootstrap-shim.rs | 56 +++++++- src/bootstrap/builder.rs | 1 - src/bootstrap/check.rs | 1 - src/bootstrap/config.rs | 21 +-- src/bootstrap/config/tests.rs | 8 +- src/bootstrap/dist.rs | 1 - src/bootstrap/download.rs | 208 ++++++++++++++-------------- src/bootstrap/min_config.rs | 20 ++- src/tools/x/src/main.rs | 30 ++-- 9 files changed, 200 insertions(+), 146 deletions(-) diff --git a/src/bootstrap/bin/bootstrap-shim.rs b/src/bootstrap/bin/bootstrap-shim.rs index d9a92ecda03fe..d44ca8f994716 100644 --- a/src/bootstrap/bin/bootstrap-shim.rs +++ b/src/bootstrap/bin/bootstrap-shim.rs @@ -1,9 +1,56 @@ -use std::{env, process::Command}; +use std::{ + env, io, + process::{self, Command, ExitStatus}, +}; use bootstrap::{Flags, MinimalConfig}; #[path = "../../../src/tools/x/src/main.rs"] -mod run_python; +mod x; + +/// We are planning to exclude python logic from x script by executing bootstrap-shim +/// immediately. Since `find_and_run_available_bootstrap_script` executes x script first, +/// any changes on bootstrap will not be seen. To prevent this problem, in bootstrap-shim +/// we want to call the python script directly. +fn find_and_run_py_bootstrap_script() { + #[cfg(unix)] + fn exec_or_status(command: &mut Command) -> io::Result { + use std::os::unix::process::CommandExt; + Err(command.exec()) + } + + #[cfg(not(unix))] + fn exec_or_status(command: &mut Command) -> io::Result { + command.status() + } + + let current_path = match env::current_dir() { + Ok(dir) => dir, + Err(err) => { + eprintln!("Failed to get current directory: {err}"); + process::exit(1); + } + }; + + for dir in current_path.ancestors() { + let candidate = dir.join("x.py"); + if candidate.exists() { + let mut cmd: Command; + cmd = Command::new(x::python()); + cmd.arg(&candidate).args(env::args().skip(1)).current_dir(dir); + let result = exec_or_status(&mut cmd); + + match result { + Err(error) => { + eprintln!("Failed to invoke `{:?}`: {}", cmd, error); + } + Ok(status) => { + process::exit(status.code().unwrap_or(1)); + } + } + } + } +} fn main() { let args = env::args().skip(1).collect::>(); @@ -15,16 +62,17 @@ fn main() { let bootstrap_bin = if let Some(commit) = last_modified_bootstrap_commit(&config) { config.download_bootstrap(&commit) } else { - return run_python::main(); + return find_and_run_py_bootstrap_script(); }; let args: Vec<_> = std::env::args().skip(1).collect(); + println!("Running pre-compiled bootstrap binary"); Command::new(bootstrap_bin).args(args).status().expect("failed to spawn bootstrap binairy"); } fn last_modified_bootstrap_commit(config: &MinimalConfig) -> Option { config.last_modified_commit( - &["src/bootstrap", "src/tools/build_helper"], + &["src/bootstrap", "src/tools/build_helper", "src/tools/x"], "download-bootstrap", true, ) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 18b5b8146bea3..a07e050912c8e 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -704,7 +704,6 @@ impl<'a> Builder<'a> { check::RustAnalyzer, check::Rustfmt, check::Bootstrap, - check::BootstrapShim, ), Kind::Test => describe!( crate::toolstate::ToolStateCheck, diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 2271034a13244..74a59907e93e0 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -450,7 +450,6 @@ tool_check_step!(MiroptTestTools, "src/tools/miropt-test-tools", SourceType::InT // FIXME: currently these are marked as ToolRustc, but they should be ToolBootstrap instead to avoid having to build the compiler first tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false); -tool_check_step!(BootstrapShim, "src/bootstrap/bin/bootstrap-shim", SourceType::InTree, false); /// Cargo's output path for the standard library in a given stage, compiled /// by a particular compiler for the specified target. diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 25760fb86ab0b..4e139a916db50 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -85,7 +85,6 @@ pub struct Config { pub jobs: Option, pub cmd: Subcommand, pub incremental: bool, - pub dry_run: DryRun, /// Arguments appearing after `--` to be forwarded to tools, /// e.g. `--fix-broken` or test arguments. pub free_args: Vec, @@ -395,7 +394,7 @@ impl Target { /// `Config` structure. #[derive(Deserialize, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] -pub struct TomlConfig { +struct TomlConfig { changelog_seen: Option, build: Option, install: Option, @@ -774,17 +773,19 @@ impl Config { args } - pub fn parse(args: &[String], custom_toml_config: Option) -> Config { + pub fn parse(args: &[String], custom_toml_config: Option<&str>) -> Config { let mut flags = Flags::parse(&args); let mut config = Config::default_opts(); - let mut toml: TomlConfig = custom_toml_config.unwrap_or_else(|| { + let mut toml: TomlConfig = if let Some(custom_toml_config) = custom_toml_config { + toml::from_str(custom_toml_config).unwrap() + } else { set_cfg_path_and_return_toml_cfg( config.src.clone(), flags.config.clone(), &mut config.config, ) - }); + }; config.minimal_config = MinimalConfig::parse(&flags, toml.build.clone()); @@ -814,11 +815,6 @@ impl Config { crate::detail_exit(1); } - let build = toml.build.clone().unwrap_or_default(); - if let Some(file_build) = build.build.as_ref() { - config.build = TargetSelection::from_user(file_build); - }; - if let Some(include) = &toml.profile { let mut include_path = config.src.clone(); include_path.push("src"); @@ -831,6 +827,11 @@ impl Config { config.changelog_seen = toml.changelog_seen; + let build = toml.build.unwrap_or_default(); + if let Some(file_build) = build.build { + config.build = TargetSelection::from_user(&file_build); + }; + set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from))); // NOTE: Bootstrap spawns various commands with different working directories. // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. diff --git a/src/bootstrap/config/tests.rs b/src/bootstrap/config/tests.rs index 3815d6517be42..e79befc716263 100644 --- a/src/bootstrap/config/tests.rs +++ b/src/bootstrap/config/tests.rs @@ -1,13 +1,9 @@ -use super::{Config, Flags, TomlConfig}; +use super::{Config, Flags}; use clap::CommandFactory; use std::{env, path::Path}; -fn toml(config: &str) -> TomlConfig { - toml::from_str(config).unwrap() -} - fn parse(config: &str) -> Config { - Config::parse(&["check".to_owned(), "--config=/does/not/exist".to_owned()], Some(toml(config))) + Config::parse(&["check".to_owned(), "--config=/does/not/exist".to_owned()], Some(config)) } #[test] diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 37d472e5689fb..78a069af03c32 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2327,7 +2327,6 @@ impl Step for BootstrapShim { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // Create this even with only `dist bootstrap` to avoid having to update all CI builders. run.alias("bootstrap-shim") } diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index a4d2f868d0da0..d0efd836cb0f5 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -359,6 +359,110 @@ enum DownloadSource { Dist, } +impl MinimalConfig { + pub fn download_bootstrap(&self, commit: &str) -> PathBuf { + self.verbose(&format!("downloading bootstrap from CI (commit {commit})")); + let host = self.build.triple; + let bin_root = self.out.join(host).join("bootstrap"); + let stamp = bin_root.join(".bootstrap-stamp"); + let bootstrap_bin = bin_root.join("bin").join("bootstrap"); + + if !bootstrap_bin.exists() || program_out_of_date(&stamp, commit) { + let version = self.git_artifact_version_part(commit); + let filename = format!("bootstrap-{version}-{host}.tar.xz"); + self.download_component(DownloadSource::CI, filename, "bootstrap", commit, ""); + if self.should_fix_bins_and_dylibs() { + self.fix_bin_or_dylib(&bootstrap_bin); + } + t!(fs::write(stamp, commit)); + } + + bootstrap_bin + } + + fn download_component( + &self, + mode: DownloadSource, + filename: String, + prefix: &str, + key: &str, + destination: &str, + ) { + let cache_dst = self.out.join("cache"); + let cache_dir = cache_dst.join(key); + if !cache_dir.exists() { + t!(fs::create_dir_all(&cache_dir)); + } + + let bin_root = self.out.join(self.build.triple).join(destination); + let tarball = cache_dir.join(&filename); + let (base_url, url, should_verify) = match mode { + DownloadSource::CI => ( + self.stage0_metadata.config.artifacts_server.clone(), + format!("{key}/{filename}"), + false, + ), + DownloadSource::Dist => { + let dist_server = env::var("RUSTUP_DIST_SERVER") + .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); + // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json + (dist_server, format!("dist/{key}/{filename}"), true) + } + }; + + // For the beta compiler, put special effort into ensuring the checksums are valid. + // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update + // this on each and every nightly ... + let checksum = if should_verify { + let error = format!( + "src/stage0.json doesn't contain a checksum for {url}. \ + Pre-built artifacts might not be available for this \ + target at this time, see https://doc.rust-lang.org/nightly\ + /rustc/platform-support.html for more information." + ); + let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error); + if tarball.exists() { + if self.verify(&tarball, sha256) { + self.unpack(&tarball, &bin_root, prefix); + return; + } else { + self.verbose(&format!( + "ignoring cached file {} due to failed verification", + tarball.display() + )); + self.remove(&tarball); + } + } + Some(sha256) + } else if tarball.exists() { + self.unpack(&tarball, &bin_root, prefix); + return; + } else { + None + }; + + let mut help_on_error = ""; + if destination == "ci-rustc" { + help_on_error = "error: failed to download pre-built rustc from CI + +note: old builds get deleted after a certain time +help: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: + +[rust] +download-rustc = false +"; + } + self.download_file(&format!("{base_url}/{url}"), &tarball, help_on_error); + if let Some(sha256) = checksum { + if !self.verify(&tarball, sha256) { + panic!("failed to verify {}", tarball.display()); + } + } + + self.unpack(&tarball, &bin_root, prefix); + } +} + /// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions. impl Config { pub(crate) fn maybe_download_rustfmt(&self) -> Option { @@ -575,107 +679,3 @@ impl Config { self.unpack(&tarball, &llvm_root, "rust-dev"); } } - -impl MinimalConfig { - pub fn download_bootstrap(&self, commit: &str) -> PathBuf { - self.verbose(&format!("downloading bootstrap from CI (commit {commit})")); - let host = self.build.triple; - let bin_root = self.out.join(host).join("bootstrap"); - let stamp = bin_root.join(".bootstrap-stamp"); - let bootstrap_bin = bin_root.join("bin").join("bootstrap"); - - if !bootstrap_bin.exists() || program_out_of_date(&stamp, commit) { - let version = self.git_artifact_version_part(commit); - let filename = format!("bootstrap-{version}-{host}.tar.xz"); - self.download_component(DownloadSource::CI, filename, "bootstrap", commit, ""); - if self.should_fix_bins_and_dylibs() { - self.fix_bin_or_dylib(&bootstrap_bin); - } - t!(fs::write(stamp, commit)); - } - - bootstrap_bin - } - - fn download_component( - &self, - mode: DownloadSource, - filename: String, - prefix: &str, - key: &str, - destination: &str, - ) { - let cache_dst = self.out.join("cache"); - let cache_dir = cache_dst.join(key); - if !cache_dir.exists() { - t!(fs::create_dir_all(&cache_dir)); - } - - let bin_root = self.out.join(self.build.triple).join(destination); - let tarball = cache_dir.join(&filename); - let (base_url, url, should_verify) = match mode { - DownloadSource::CI => ( - self.stage0_metadata.config.artifacts_server.clone(), - format!("{key}/{filename}"), - false, - ), - DownloadSource::Dist => { - let dist_server = env::var("RUSTUP_DIST_SERVER") - .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); - // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json - (dist_server, format!("dist/{key}/{filename}"), true) - } - }; - - // For the beta compiler, put special effort into ensuring the checksums are valid. - // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update - // this on each and every nightly ... - let checksum = if should_verify { - let error = format!( - "src/stage0.json doesn't contain a checksum for {url}. \ - Pre-built artifacts might not be available for this \ - target at this time, see https://doc.rust-lang.org/nightly\ - /rustc/platform-support.html for more information." - ); - let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error); - if tarball.exists() { - if self.verify(&tarball, sha256) { - self.unpack(&tarball, &bin_root, prefix); - return; - } else { - self.verbose(&format!( - "ignoring cached file {} due to failed verification", - tarball.display() - )); - self.remove(&tarball); - } - } - Some(sha256) - } else if tarball.exists() { - self.unpack(&tarball, &bin_root, prefix); - return; - } else { - None - }; - - let mut help_on_error = ""; - if destination == "ci-rustc" { - help_on_error = "error: failed to download pre-built rustc from CI - -note: old builds get deleted after a certain time -help: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: - -[rust] -download-rustc = false -"; - } - self.download_file(&format!("{base_url}/{url}"), &tarball, help_on_error); - if let Some(sha256) = checksum { - if !self.verify(&tarball, sha256) { - panic!("failed to verify {}", tarball.display()); - } - } - - self.unpack(&tarball, &bin_root, prefix); - } -} diff --git a/src/bootstrap/min_config.rs b/src/bootstrap/min_config.rs index 9282cf609c928..5ee031192ecc5 100644 --- a/src/bootstrap/min_config.rs +++ b/src/bootstrap/min_config.rs @@ -7,7 +7,6 @@ use std::{ process::Command, }; -use serde::Deserialize; use serde_derive::Deserialize; use crate::{ @@ -19,6 +18,12 @@ use crate::{ }; /// The bare minimum config, suitable for `bootstrap-shim`, but sharing code with the main `bootstrap` binary. +/// Unlike bootstrap itself, bootstrap-shim needs to be compatible across multiple versions of the Rust repo. +/// +/// FIXME: Update the following information once bootstrap-shim is default in shell scripts +/// +/// Once bootstrap-shim is used as default(today it's opt in) in shell scripts, DO NOT CHANGE this configuration +/// without the version bump for the pinned nightly version of bootstrap-shim. #[derive(Default, Clone)] pub struct MinimalConfig { // Needed so we know where to store the unpacked bootstrap binary. @@ -30,12 +35,11 @@ pub struct MinimalConfig { pub patch_binaries_for_nix: bool, // Needed to know which commit to download. pub stage0_metadata: Stage0Metadata, - // This isn't currently used, but will eventually let people configure whether to download or build bootstrap. pub config: Option, - // Not currently used in the shim. + // General need for verbose mode logging in `bootstrap-shim`. pub verbose: usize, - // Not currently used in the shim. + // Needed for `bootstrap::download` module pub dry_run: DryRun, } @@ -46,11 +50,13 @@ pub struct Stage0Metadata { pub checksums_sha256: HashMap, pub rustfmt: Option, } + #[derive(Clone, Default, Deserialize)] pub struct CompilerMetadata { pub date: String, pub version: String, } + #[derive(Default, Deserialize, Clone)] pub struct Stage0Config { pub dist_server: String, @@ -228,12 +234,12 @@ impl MinimalConfig { #[cfg(test)] /// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` -pub(crate) fn get_toml + Default>(_file: &Path) -> T { +pub(crate) fn get_toml + Default>(_file: &Path) -> T { T::default() } #[cfg(not(test))] /// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` -pub(crate) fn get_toml + Default>(file: &Path) -> T { +pub(crate) fn get_toml + Default>(file: &Path) -> T { let contents = t!(fs::read_to_string(file), format!("config file {} not found", file.display())); // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of @@ -262,7 +268,7 @@ fn set_config_output_dir(output_path: &mut PathBuf) { } /// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` -pub(crate) fn set_cfg_path_and_return_toml_cfg + Default>( +pub(crate) fn set_cfg_path_and_return_toml_cfg + Default>( src: PathBuf, config_flag: Option, cfg_path: &mut Option, diff --git a/src/tools/x/src/main.rs b/src/tools/x/src/main.rs index 344cb3e48b76f..bb506ca4ebf4f 100644 --- a/src/tools/x/src/main.rs +++ b/src/tools/x/src/main.rs @@ -19,7 +19,7 @@ const PYTHON: &str = "python"; const PYTHON2: &str = "python2"; const PYTHON3: &str = "python3"; -fn python() -> &'static str { +pub fn python() -> &'static str { let val = match env::var_os("PATH") { Some(val) => val, None => return PYTHON, @@ -98,16 +98,8 @@ fn handle_result(result: io::Result, cmd: Command) { } } -pub fn main() { - match env::args().skip(1).next().as_deref() { - Some("--wrapper-version") => { - let version = env!("CARGO_PKG_VERSION"); - println!("{}", version); - return; - } - _ => {} - } - let current = match env::current_dir() { +fn find_and_run_available_bootstrap_script() { + let current_path = match env::current_dir() { Ok(dir) => dir, Err(err) => { eprintln!("Failed to get current directory: {err}"); @@ -115,7 +107,7 @@ pub fn main() { } }; - for dir in current.ancestors() { + for dir in current_path.ancestors() { let candidate = dir.join("x.py"); if candidate.exists() { let shell_script_candidate = dir.join("x"); @@ -132,6 +124,20 @@ pub fn main() { handle_result(result, cmd); } } +} + +#[allow(dead_code)] +fn main() { + match env::args().skip(1).next().as_deref() { + Some("--wrapper-version") => { + let version = env!("CARGO_PKG_VERSION"); + println!("{}", version); + return; + } + _ => {} + } + + find_and_run_available_bootstrap_script(); eprintln!( "x.py not found. Please run inside of a checkout of `https://github.com/rust-lang/rust`."