From 36c314cfb7999a66a49932822a336aea47f62292 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 14 Feb 2025 11:16:32 +0100 Subject: [PATCH 1/4] generate-copyright: pass the source root from bootstrap --- src/bootstrap/src/core/build_steps/run.rs | 1 + src/tools/generate-copyright/src/main.rs | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 167b8a5b168c9..84c94a268a38c 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -217,6 +217,7 @@ impl Step for GenerateCopyright { cmd.env("DEST", &dest); cmd.env("DEST_LIBSTD", &dest_libstd); cmd.env("OUT_DIR", &builder.out); + cmd.env("SRC_DIR", &builder.src); cmd.env("CARGO", &builder.initial_cargo); // it is important that generate-copyright runs from the root of the // source tree, because it uses relative paths diff --git a/src/tools/generate-copyright/src/main.rs b/src/tools/generate-copyright/src/main.rs index 7a014989e6880..677ac76439e88 100644 --- a/src/tools/generate-copyright/src/main.rs +++ b/src/tools/generate-copyright/src/main.rs @@ -18,6 +18,7 @@ fn main() -> Result<(), Error> { let dest_file = env_path("DEST")?; let libstd_dest_file = env_path("DEST_LIBSTD")?; let out_dir = env_path("OUT_DIR")?; + let src_dir = env_path("SRC_DIR")?; let cargo = env_path("CARGO")?; let license_metadata = env_path("LICENSE_METADATA")?; @@ -27,7 +28,7 @@ fn main() -> Result<(), Error> { let mut collected_cargo_metadata = cargo_metadata::get_metadata_and_notices( &cargo, &out_dir.join("vendor"), - &root_path, + &src_dir, &[ Path::new("./Cargo.toml"), Path::new("./src/tools/cargo/Cargo.toml"), @@ -38,7 +39,7 @@ fn main() -> Result<(), Error> { let library_collected_cargo_metadata = cargo_metadata::get_metadata_and_notices( &cargo, &out_dir.join("library-vendor"), - &root_path, + &src_dir, &[Path::new("./library/Cargo.toml")], )?; @@ -54,7 +55,7 @@ fn main() -> Result<(), Error> { let library_collected_tree_metadata = Metadata { files: collected_tree_metadata .files - .trim_clone(&Path::new("./library"), &Path::new(".")) + .trim_clone(&src_dir.join("library"), &src_dir) .unwrap(), }; From 08b4f6d2c650d3e6e9010e8a27631962bf31dec7 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 14 Feb 2025 11:16:47 +0100 Subject: [PATCH 2/4] generate-copyright: pass the list of manifests from bootstrap --- src/bootstrap/src/core/build_steps/run.rs | 15 ++++++++ .../generate-copyright/src/cargo_metadata.rs | 6 ++-- src/tools/generate-copyright/src/main.rs | 35 +++++++++++++++---- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 84c94a268a38c..3f5e701e7e17a 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -9,6 +9,7 @@ use crate::Mode; use crate::core::build_steps::dist::distdir; use crate::core::build_steps::test; use crate::core::build_steps::tool::{self, SourceType, Tool}; +use crate::core::build_steps::vendor::default_paths_to_vendor; use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; use crate::core::config::flags::get_completion; @@ -212,7 +213,21 @@ impl Step for GenerateCopyright { let dest = builder.out.join("COPYRIGHT.html"); let dest_libstd = builder.out.join("COPYRIGHT-library.html"); + let paths_to_vendor = default_paths_to_vendor(builder); + for (_, submodules) in &paths_to_vendor { + for submodule in submodules { + builder.build.require_submodule(submodule, None); + } + } + let cargo_manifests = paths_to_vendor + .into_iter() + .map(|(path, _submodules)| path.to_str().unwrap().to_string()) + .inspect(|path| assert!(!path.contains(','), "{path} contains a comma in its name")) + .collect::>() + .join(","); + let mut cmd = builder.tool_cmd(Tool::GenerateCopyright); + cmd.env("CARGO_MANIFESTS", &cargo_manifests); cmd.env("LICENSE_METADATA", &license_metadata); cmd.env("DEST", &dest); cmd.env("DEST_LIBSTD", &dest_libstd); diff --git a/src/tools/generate-copyright/src/cargo_metadata.rs b/src/tools/generate-copyright/src/cargo_metadata.rs index 51e353e9b229f..16c5b5e7104e2 100644 --- a/src/tools/generate-copyright/src/cargo_metadata.rs +++ b/src/tools/generate-copyright/src/cargo_metadata.rs @@ -54,7 +54,7 @@ pub fn get_metadata_and_notices( cargo: &Path, vendor_path: &Path, root_path: &Path, - manifest_paths: &[&Path], + manifest_paths: &[PathBuf], ) -> Result, Error> { let mut output = get_metadata(cargo, root_path, manifest_paths)?; @@ -77,7 +77,7 @@ pub fn get_metadata_and_notices( pub fn get_metadata( cargo: &Path, root_path: &Path, - manifest_paths: &[&Path], + manifest_paths: &[PathBuf], ) -> Result, Error> { let mut output = BTreeMap::new(); // Look at the metadata for each manifest @@ -114,7 +114,7 @@ pub fn get_metadata( } /// Run cargo-vendor, fetching into the given dir -fn run_cargo_vendor(cargo: &Path, dest: &Path, manifest_paths: &[&Path]) -> Result<(), Error> { +fn run_cargo_vendor(cargo: &Path, dest: &Path, manifest_paths: &[PathBuf]) -> Result<(), Error> { let mut vendor_command = std::process::Command::new(cargo); vendor_command.env("RUSTC_BOOTSTRAP", "1"); vendor_command.arg("vendor"); diff --git a/src/tools/generate-copyright/src/main.rs b/src/tools/generate-copyright/src/main.rs index 677ac76439e88..7b7cf0f4b6991 100644 --- a/src/tools/generate-copyright/src/main.rs +++ b/src/tools/generate-copyright/src/main.rs @@ -22,25 +22,35 @@ fn main() -> Result<(), Error> { let cargo = env_path("CARGO")?; let license_metadata = env_path("LICENSE_METADATA")?; - let root_path = std::path::absolute(".")?; + let cargo_manifests = env_string("CARGO_MANIFESTS")? + .split(",") + .map(|manifest| manifest.into()) + .collect::>(); + let library_manifests = cargo_manifests + .iter() + .filter(|path| { + if let Ok(stripped) = path.strip_prefix(&src_dir) { + stripped.starts_with("library") + } else { + panic!("manifest {path:?} not relative to source dir {src_dir:?}"); + } + }) + .cloned() + .collect::>(); // Scan Cargo dependencies let mut collected_cargo_metadata = cargo_metadata::get_metadata_and_notices( &cargo, &out_dir.join("vendor"), &src_dir, - &[ - Path::new("./Cargo.toml"), - Path::new("./src/tools/cargo/Cargo.toml"), - Path::new("./library/Cargo.toml"), - ], + &cargo_manifests, )?; let library_collected_cargo_metadata = cargo_metadata::get_metadata_and_notices( &cargo, &out_dir.join("library-vendor"), &src_dir, - &[Path::new("./library/Cargo.toml")], + &library_manifests, )?; for (key, value) in collected_cargo_metadata.iter_mut() { @@ -194,6 +204,17 @@ struct License { copyright: Vec, } +/// Grab an environment variable as string, or fail nicely. +fn env_string(var: &str) -> Result { + match std::env::var(var) { + Ok(var) => Ok(var), + Err(std::env::VarError::NotUnicode(_)) => { + anyhow::bail!("environment variable {var} is not utf-8") + } + Err(std::env::VarError::NotPresent) => anyhow::bail!("missing environment variable {var}"), + } +} + /// Grab an environment variable as a PathBuf, or fail nicely. fn env_path(var: &str) -> Result { if let Some(var) = std::env::var_os(var) { From 33e7f9bc6681e32cc3b3b26ea85e21da225b89ee Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 17 Feb 2025 10:20:14 +0100 Subject: [PATCH 3/4] generate-copyright: pass the vendored sources from bootstrap --- src/bootstrap/src/core/build_steps/run.rs | 17 +++++++- src/bootstrap/src/core/build_steps/vendor.rs | 14 +++++-- src/bootstrap/src/core/builder/cargo.rs | 3 +- src/bootstrap/src/lib.rs | 7 ++++ .../generate-copyright/src/cargo_metadata.rs | 40 ++----------------- src/tools/generate-copyright/src/main.rs | 12 ++---- 6 files changed, 41 insertions(+), 52 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 3f5e701e7e17a..2b17e02cae5a4 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -9,7 +9,7 @@ use crate::Mode; use crate::core::build_steps::dist::distdir; use crate::core::build_steps::test; use crate::core::build_steps::tool::{self, SourceType, Tool}; -use crate::core::build_steps::vendor::default_paths_to_vendor; +use crate::core::build_steps::vendor::{Vendor, default_paths_to_vendor}; use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; use crate::core::config::flags::get_completion; @@ -226,13 +226,26 @@ impl Step for GenerateCopyright { .collect::>() .join(","); + let vendored_sources = if let Some(path) = builder.vendored_crates_path() { + path + } else { + let cache_dir = builder.out.join("tmp").join("generate-copyright-vendor"); + builder.ensure(Vendor { + sync_args: Vec::new(), + versioned_dirs: true, + root_dir: builder.src.clone(), + output_dir: cache_dir.clone(), + }); + cache_dir + }; + let mut cmd = builder.tool_cmd(Tool::GenerateCopyright); cmd.env("CARGO_MANIFESTS", &cargo_manifests); cmd.env("LICENSE_METADATA", &license_metadata); cmd.env("DEST", &dest); cmd.env("DEST_LIBSTD", &dest_libstd); - cmd.env("OUT_DIR", &builder.out); cmd.env("SRC_DIR", &builder.src); + cmd.env("VENDOR_DIR", &vendored_sources); cmd.env("CARGO", &builder.initial_cargo); // it is important that generate-copyright runs from the root of the // source tree, because it uses relative paths diff --git a/src/bootstrap/src/core/build_steps/vendor.rs b/src/bootstrap/src/core/build_steps/vendor.rs index 26d0f100ffd52..c68b55f358941 100644 --- a/src/bootstrap/src/core/build_steps/vendor.rs +++ b/src/bootstrap/src/core/build_steps/vendor.rs @@ -4,6 +4,8 @@ use crate::core::build_steps::tool::SUBMODULES_FOR_RUSTBOOK; use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::utils::exec::command; +pub const VENDOR_DIR: &str = "vendor"; + /// Returns the cargo workspaces to vendor for `x vendor` and dist tarballs. /// /// Returns a `Vec` of `(path_to_manifest, submodules_required)` where @@ -29,9 +31,10 @@ pub fn default_paths_to_vendor(builder: &Builder<'_>) -> Vec<(PathBuf, Vec<&'sta #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub(crate) struct Vendor { - sync_args: Vec, - versioned_dirs: bool, - root_dir: PathBuf, + pub(crate) sync_args: Vec, + pub(crate) versioned_dirs: bool, + pub(crate) root_dir: PathBuf, + pub(crate) output_dir: PathBuf, } impl Step for Vendor { @@ -48,10 +51,13 @@ impl Step for Vendor { sync_args: run.builder.config.cmd.vendor_sync_args(), versioned_dirs: run.builder.config.cmd.vendor_versioned_dirs(), root_dir: run.builder.src.clone(), + output_dir: run.builder.src.join(VENDOR_DIR), }); } fn run(self, builder: &Builder<'_>) -> Self::Output { + builder.info(&format!("Vendoring sources to {:?}", self.root_dir)); + let mut cmd = command(&builder.initial_cargo); cmd.arg("vendor"); @@ -81,7 +87,7 @@ impl Step for Vendor { // which uses the unstable `public-dependency` feature. cmd.env("RUSTC_BOOTSTRAP", "1"); - cmd.current_dir(self.root_dir); + cmd.current_dir(self.root_dir).arg(&self.output_dir); cmd.run(builder); } diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index 59680af00622f..1ec3e601cad19 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -924,8 +924,7 @@ impl Builder<'_> { if self.config.rust_remap_debuginfo { let mut env_var = OsString::new(); - if self.config.vendor { - let vendor = self.build.src.join("vendor"); + if let Some(vendor) = self.build.vendored_crates_path() { env_var.push(vendor); env_var.push("=/rust/deps"); } else { diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 7cd8aacf0d6c8..e4a6e2b55e73d 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -50,6 +50,8 @@ pub use utils::change_tracker::{ CONFIG_CHANGE_HISTORY, find_recent_config_change_ids, human_readable_changes, }; +use crate::core::build_steps::vendor::VENDOR_DIR; + const LLVM_TOOLS: &[&str] = &[ "llvm-cov", // used to generate coverage report "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility @@ -782,6 +784,11 @@ impl Build { self.out.join(target).join("md-doc") } + /// Path to the vendored Rust crates. + fn vendored_crates_path(&self) -> Option { + if self.config.vendor { Some(self.src.join(VENDOR_DIR)) } else { None } + } + /// Returns `true` if this is an external version of LLVM not managed by bootstrap. /// In particular, we expect llvm sources to be available when this is false. /// diff --git a/src/tools/generate-copyright/src/cargo_metadata.rs b/src/tools/generate-copyright/src/cargo_metadata.rs index 16c5b5e7104e2..b717bd53eb1a7 100644 --- a/src/tools/generate-copyright/src/cargo_metadata.rs +++ b/src/tools/generate-copyright/src/cargo_metadata.rs @@ -11,10 +11,6 @@ pub enum Error { Io(#[from] std::io::Error), #[error("Failed get output from cargo-metadata: {0:?}")] GettingMetadata(#[from] cargo_metadata::Error), - #[error("Failed to run cargo vendor: {0:?}")] - LaunchingVendor(std::io::Error), - #[error("Failed to complete cargo vendor")] - RunningVendor, #[error("Bad path {0:?} whilst scraping files")] Scraping(PathBuf), } @@ -43,13 +39,11 @@ pub struct PackageMetadata { pub is_in_libstd: Option, } -/// Use `cargo metadata` and `cargo vendor` to get a list of dependencies and their license data. +/// Use `cargo metadata` to get a list of dependencies and their license data. License files will +/// also be pulled from the vendor path (generated by bootstrap). /// -/// This will involve running `cargo vendor` into `vendor_path` so we can -/// grab the license files. -/// -/// Any dependency with a path beginning with `root_path` is ignored, as we -/// assume `reuse` has covered it already. +/// Any dependency with a path beginning with `root_path` is ignored, as we assume `reuse` has +/// covered it already. pub fn get_metadata_and_notices( cargo: &Path, vendor_path: &Path, @@ -58,10 +52,6 @@ pub fn get_metadata_and_notices( ) -> Result, Error> { let mut output = get_metadata(cargo, root_path, manifest_paths)?; - // Now do a cargo-vendor and grab everything - println!("Vendoring deps into {}...", vendor_path.display()); - run_cargo_vendor(cargo, &vendor_path, manifest_paths)?; - // Now for each dependency we found, go and grab any important looking files for (package, metadata) in output.iter_mut() { load_important_files(package, metadata, &vendor_path)?; @@ -113,28 +103,6 @@ pub fn get_metadata( Ok(output) } -/// Run cargo-vendor, fetching into the given dir -fn run_cargo_vendor(cargo: &Path, dest: &Path, manifest_paths: &[PathBuf]) -> Result<(), Error> { - let mut vendor_command = std::process::Command::new(cargo); - vendor_command.env("RUSTC_BOOTSTRAP", "1"); - vendor_command.arg("vendor"); - vendor_command.arg("--quiet"); - vendor_command.arg("--versioned-dirs"); - for manifest_path in manifest_paths { - vendor_command.arg("-s"); - vendor_command.arg(manifest_path); - } - vendor_command.arg(dest); - - let vendor_status = vendor_command.status().map_err(Error::LaunchingVendor)?; - - if !vendor_status.success() { - return Err(Error::RunningVendor); - } - - Ok(()) -} - /// Add important files off disk into this dependency. /// /// Maybe one-day Cargo.toml will contain enough information that we don't need diff --git a/src/tools/generate-copyright/src/main.rs b/src/tools/generate-copyright/src/main.rs index 7b7cf0f4b6991..79e90d88f4446 100644 --- a/src/tools/generate-copyright/src/main.rs +++ b/src/tools/generate-copyright/src/main.rs @@ -17,8 +17,8 @@ mod cargo_metadata; fn main() -> Result<(), Error> { let dest_file = env_path("DEST")?; let libstd_dest_file = env_path("DEST_LIBSTD")?; - let out_dir = env_path("OUT_DIR")?; let src_dir = env_path("SRC_DIR")?; + let vendor_dir = env_path("VENDOR_DIR")?; let cargo = env_path("CARGO")?; let license_metadata = env_path("LICENSE_METADATA")?; @@ -39,16 +39,12 @@ fn main() -> Result<(), Error> { .collect::>(); // Scan Cargo dependencies - let mut collected_cargo_metadata = cargo_metadata::get_metadata_and_notices( - &cargo, - &out_dir.join("vendor"), - &src_dir, - &cargo_manifests, - )?; + let mut collected_cargo_metadata = + cargo_metadata::get_metadata_and_notices(&cargo, &vendor_dir, &src_dir, &cargo_manifests)?; let library_collected_cargo_metadata = cargo_metadata::get_metadata_and_notices( &cargo, - &out_dir.join("library-vendor"), + &vendor_dir, &src_dir, &library_manifests, )?; From 92f31b95c92cb0a34413a52793a50c7d4334dbb2 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 17 Feb 2025 10:29:51 +0100 Subject: [PATCH 4/4] use the shared vendor impl for plan source tarballs --- src/bootstrap/src/core/build_steps/dist.rs | 28 +++++++------------- src/bootstrap/src/core/build_steps/vendor.rs | 10 +++++-- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index ae3761a97e50f..c33f11f684f4d 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -19,7 +19,7 @@ use object::read::archive::ArchiveFile; use crate::core::build_steps::doc::DocumentationFormat; use crate::core::build_steps::tool::{self, Tool}; -use crate::core::build_steps::vendor::default_paths_to_vendor; +use crate::core::build_steps::vendor::{VENDOR_DIR, Vendor}; use crate::core::build_steps::{compile, llvm}; use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; @@ -1050,19 +1050,6 @@ impl Step for PlainSourceTarball { if builder.config.dist_vendor { builder.require_and_update_all_submodules(); - // Vendor all Cargo dependencies - let mut cmd = command(&builder.initial_cargo); - cmd.arg("vendor").arg("--versioned-dirs"); - - for (p, _) in default_paths_to_vendor(builder) { - cmd.arg("--sync").arg(p); - } - - cmd - // Will read the libstd Cargo.toml which uses the unstable `public-dependency` feature. - .env("RUSTC_BOOTSTRAP", "1") - .current_dir(plain_dst_src); - // Vendor packages that are required by opt-dist to collect PGO profiles. let pkgs_for_pgo_training = build_helper::LLVM_PGO_CRATES .iter() @@ -1074,15 +1061,18 @@ impl Step for PlainSourceTarball { manifest_path.push("Cargo.toml"); manifest_path }); - for manifest_path in pkgs_for_pgo_training { - cmd.arg("--sync").arg(manifest_path); - } - let config = cmd.run_capture(builder).stdout(); + // Vendor all Cargo dependencies + let vendor = builder.ensure(Vendor { + sync_args: pkgs_for_pgo_training.collect(), + versioned_dirs: true, + root_dir: plain_dst_src.into(), + output_dir: VENDOR_DIR.into(), + }); let cargo_config_dir = plain_dst_src.join(".cargo"); builder.create_dir(&cargo_config_dir); - builder.create(&cargo_config_dir.join("config.toml"), &config); + builder.create(&cargo_config_dir.join("config.toml"), &vendor.config); } // Delete extraneous directories diff --git a/src/bootstrap/src/core/build_steps/vendor.rs b/src/bootstrap/src/core/build_steps/vendor.rs index c68b55f358941..410dbc04f0306 100644 --- a/src/bootstrap/src/core/build_steps/vendor.rs +++ b/src/bootstrap/src/core/build_steps/vendor.rs @@ -38,7 +38,7 @@ pub(crate) struct Vendor { } impl Step for Vendor { - type Output = (); + type Output = VendorOutput; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -89,6 +89,12 @@ impl Step for Vendor { cmd.current_dir(self.root_dir).arg(&self.output_dir); - cmd.run(builder); + let config = cmd.run_capture_stdout(builder); + VendorOutput { config: config.stdout() } } } + +#[derive(Debug, Clone)] +pub(crate) struct VendorOutput { + pub(crate) config: String, +}