diff --git a/Cargo.lock b/Cargo.lock index 7e2aa87310..81e071a2ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -37,6 +46,9 @@ name = "anyhow" version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +dependencies = [ + "backtrace", +] [[package]] name = "ar" @@ -131,6 +143,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "backtrace" +version = "0.3.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.13.1" @@ -475,9 +502,9 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" [[package]] name = "der" -version = "0.6.1" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +checksum = "56acb310e15652100da43d130af8d97b509e95af61aab1c5a7939ef24337ee17" dependencies = [ "const-oid", "pem-rfc7468", @@ -889,6 +916,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" + [[package]] name = "gzp" version = "0.11.3" @@ -1333,6 +1366,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -1541,6 +1583,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.30.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" +dependencies = [ + "flate2", + "memchr", +] + [[package]] name = "once_cell" version = "1.17.1" @@ -1549,9 +1601,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "opendal" -version = "0.34.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005c877c4f788a7749825bbc61031ccc950217fac6bcf9965641fbf1bdf991b0" +checksum = "3555168d4cc9a83c332e1416ff00e3be36a6d78447dff472829962afbc91bb3d" dependencies = [ "anyhow", "async-compat", @@ -1577,6 +1629,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "tokio", "uuid", ] @@ -1686,9 +1739,9 @@ dependencies = [ [[package]] name = "pem-rfc7468" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d159833a9105500e0398934e205e0773f0b27529557134ecfc51c27646adac" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" dependencies = [ "base64ct", ] @@ -1733,21 +1786,20 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkcs1" -version = "0.4.1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff33bdbdfc54cc98a2eca766ebdec3e1b8fb7387523d5c9c9a2891da856f719" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ "der", "pkcs8", "spki", - "zeroize", ] [[package]] name = "pkcs8" -version = "0.9.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", "spki", @@ -1967,9 +2019,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqsign" -version = "0.10.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d36dae58fbc1db90e1cf14ca8fabcba92b7aa3c282d5e46bcdf16b9c28ab04c" +checksum = "b04f5fccb94d61c154f0d8520ec42e79afdc145f4b1a392faa269874995fda66" dependencies = [ "anyhow", "async-trait", @@ -2101,11 +2153,12 @@ dependencies = [ [[package]] name = "rsa" -version = "0.8.2" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55a77d189da1fee555ad95b7e50e7457d91c0e089ec68ca69ad2989413bbdab4" +checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" dependencies = [ "byteorder", + "const-oid", "digest", "num-bigint-dig", "num-integer", @@ -2115,6 +2168,7 @@ dependencies = [ "pkcs8", "rand_core", "signature", + "spki", "subtle", "zeroize", ] @@ -2129,6 +2183,12 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustix" version = "0.35.13" @@ -2246,10 +2306,12 @@ dependencies = [ "libmount", "linked-hash-map", "log", + "memmap2", "mime", "nix 0.26.2", "num_cpus", "number_prefix", + "object", "once_cell", "opendal", "openssl", @@ -2268,6 +2330,7 @@ dependencies = [ "strip-ansi-escapes", "syslog", "tar", + "temp-env", "tempfile", "thirtyfour_sync", "tokio", @@ -2531,9 +2594,9 @@ dependencies = [ [[package]] name = "spki" -version = "0.6.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", "der", @@ -2621,6 +2684,15 @@ dependencies = [ "xattr", ] +[[package]] +name = "temp-env" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9547444bfe52cbd79515c6c8087d8ae6ca8d64d2d31a27746320f5cb81d1a15c" +dependencies = [ + "parking_lot", +] + [[package]] name = "tempfile" version = "3.4.0" diff --git a/Cargo.toml b/Cargo.toml index 9d80a72a2d..d24d0a0461 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ lto = true strip = true [dependencies] -anyhow = "1.0" +anyhow = { version = "1.0", features = ["backtrace"] } ar = "0.9" async-trait = "0.1" base64 = "0.21" @@ -31,8 +31,8 @@ bincode = "1" blake3 = "1" byteorder = "1.0" bytes = "1" -opendal = { version= "0.34.0", optional = true } -reqsign = {version="0.10.1", optional = true} +opendal = { version= "0.36.0", optional = true } +reqsign = { version = "0.12.0", optional = true } clap = { version = "4.1.11", features = ["derive", "env", "wrap_help"] } directories = "5.0.0" encoding = "0.2" @@ -86,6 +86,8 @@ nix = { version = "0.26.2", optional = true } rouille = { version = "3.5", optional = true, default-features = false, features = ["ssl"] } syslog = { version = "6", optional = true } version-compare = { version = "0.1.1", optional = true } +object = "0.30" +memmap2 = "0.6.2" [dev-dependencies] assert_cmd = "2.0.10" @@ -95,6 +97,7 @@ itertools = "0.10" predicates = "=3.0.2" thirtyfour_sync = "0.27" serial_test = "2.0" +temp-env = "0.3.4" [target.'cfg(unix)'.dependencies] daemonize = "0.5" diff --git a/docs/Caching.md b/docs/Caching.md index d3cc078a80..c2627dfc11 100644 --- a/docs/Caching.md +++ b/docs/Caching.md @@ -25,7 +25,10 @@ In parallel, we also take into account in the hash: For C/C++, the hash is generated with a blake3 digest of the preprocessed file (-E with gcc/clang). For compilations that specify multiple `-arch` flags, these flags are rewritten to their corresponding preprocessor defines to allow -pre-processing the file (e.g `-arch x86_64` is rewritten to `-D__X86_64__=1`). +pre-processing the file (e.g `-arch x86_64` is rewritten to `-D__X86_64__=1`), +this can be enabled by setting the environment variable +`SCCACHE_CACHE_MULTIARCH` but is disabled by default as it may not work in all +cases. We also take into account in the hash: * Hash of the compiler binary diff --git a/docs/Configuration.md b/docs/Configuration.md index 84df9f6073..85fd2b3c31 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -75,6 +75,7 @@ configuration variables * `SCCACHE_STARTUP_NOTIFY` specify a path to a socket which will be used for server completion notification * `SCCACHE_MAX_FRAME_LENGTH` how much data can be transferred between client and server * `SCCACHE_NO_DAEMON` set to `1` to disable putting the server to the background +* `SCCACHE_CACHE_MULTIARCH` to disable caching of multi architecture builds. ### cache configs diff --git a/src/compiler/gcc.rs b/src/compiler/gcc.rs index 9fd27778bf..99a1126468 100644 --- a/src/compiler/gcc.rs +++ b/src/compiler/gcc.rs @@ -25,6 +25,7 @@ use fs::File; use fs_err as fs; use log::Level::Trace; use std::collections::HashMap; +use std::env; use std::ffi::OsString; use std::io::Read; use std::path::{Path, PathBuf}; @@ -275,6 +276,8 @@ where let mut outputs_gcno = false; let mut xclangs: Vec = vec![]; let mut color_mode = ColorMode::Auto; + let mut seen_arch = None; + let dont_cache_multiarch = env::var("SCCACHE_CACHE_MULTIARCH").is_err(); // Custom iterator to expand `@` arguments which stand for reading a file // and interpreting it as a list of more arguments. @@ -365,7 +368,17 @@ where _ => cannot_cache!("-x"), }; } - Some(Arch(_)) => {} + Some(Arch(arch)) => { + match seen_arch { + Some(s) if &s != arch && dont_cache_multiarch => { + cannot_cache!( + "multiple different -arch, and SCCACHE_CACHE_MULTIARCH not set" + ) + } + _ => {} + }; + seen_arch = Some(arch.clone()); + } Some(XClang(s)) => xclangs.push(s.clone()), None => match arg { Argument::Raw(ref val) => { @@ -651,11 +664,20 @@ fn preprocess_cmd( }) .collect::>(); + let mut arch_args_to_use = &rewritten_arch_args; + let mut unique_rewritten = rewritten_arch_args.clone(); + unique_rewritten.sort(); + unique_rewritten.dedup(); + if unique_rewritten.len() <= 1 { + // don't use rewritten arch args if there is only one arch + arch_args_to_use = &parsed_args.arch_args; + } + cmd.arg(&parsed_args.input) .args(&parsed_args.preprocessor_args) .args(&parsed_args.dependency_args) .args(&parsed_args.common_args) - .args(&rewritten_arch_args) + .args(arch_args_to_use) .env_clear() .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(cwd); @@ -884,6 +906,8 @@ mod test { use crate::mock_command::*; use crate::test::utils::*; + use temp_env::with_var; + fn parse_arguments_( arguments: Vec, plusplus: bool, @@ -1428,7 +1452,43 @@ mod test { #[test] fn test_preprocess_cmd_rewrites_archs() { - let args = stringvec!["-arch", "arm64", "-arch", "i386", "-c", "foo.cc"]; + with_var("SCCACHE_CACHE_MULTIARCH", Some("1"), || { + let args = stringvec!["-arch", "arm64", "-arch", "i386", "-c", "foo.cc"]; + let parsed_args = match parse_arguments_(args, false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + let mut cmd = MockCommand { + child: None, + args: vec![], + }; + preprocess_cmd( + &mut cmd, + &parsed_args, + Path::new(""), + &[], + true, + CCompilerKind::Gcc, + true, + vec![], + ); + // make sure the architectures were rewritten to prepocessor defines + let expected_args = ovec![ + "-x", + "c++", + "-E", + "-fdirectives-only", + "foo.cc", + "-D__arm64__=1", + "-D__i386__=1" + ]; + assert_eq!(cmd.args, expected_args); + }); + } + + #[test] + fn test_preprocess_cmd_doesnt_rewrite_single_arch() { + let args = stringvec!["-arch", "arm64", "-c", "foo.cc"]; let parsed_args = match parse_arguments_(args, false) { CompilerArguments::Ok(args) => args, o => panic!("Got unexpected parse result: {:?}", o), @@ -1454,8 +1514,8 @@ mod test { "-E", "-fdirectives-only", "foo.cc", - "-D__arm64__=1", - "-D__i386__=1" + "-arch", + "arm64" ]; assert_eq!(cmd.args, expected_args); } @@ -1681,6 +1741,22 @@ mod test { ); } + #[test] + fn test_parse_arguments_multiarch_cache_disabled() { + assert_eq!( + CompilerArguments::CannotCache( + "multiple different -arch, and SCCACHE_CACHE_MULTIARCH not set", + None + ), + parse_arguments_( + stringvec![ + "-fPIC", "-arch", "arm64", "-arch", "i386", "-o", "foo.o", "-c", "foo.cpp" + ], + false + ) + ) + } + #[test] fn test_parse_arguments_multiple_arch() { match parse_arguments_( @@ -1691,47 +1767,50 @@ mod test { o => panic!("Got unexpected parse result: {:?}", o), } - match parse_arguments_( - stringvec!["-arch", "arm64", "-arch", "arm64", "-o", "foo.o", "-c", "foo.cpp"], - false, - ) { - CompilerArguments::Ok(_) => {} - o => panic!("Got unexpected parse result: {:?}", o), - } + with_var("SCCACHE_CACHE_MULTIARCH", Some("1"), || { + match parse_arguments_( + stringvec!["-arch", "arm64", "-arch", "arm64", "-o", "foo.o", "-c", "foo.cpp"], + false, + ) { + CompilerArguments::Ok(_) => {} + o => panic!("Got unexpected parse result: {:?}", o), + } - let args = - stringvec!["-fPIC", "-arch", "arm64", "-arch", "i386", "-o", "foo.o", "-c", "foo.cpp"]; - let ParsedArguments { - input, - language, - compilation_flag, - outputs, - preprocessor_args, - msvc_show_includes, - common_args, - arch_args, - .. - } = match parse_arguments_(args, false) { - CompilerArguments::Ok(args) => args, - o => panic!("Got unexpected parse result: {:?}", o), - }; - assert_eq!(Some("foo.cpp"), input.to_str()); - assert_eq!(Language::Cxx, language); - assert_eq!(Some("-c"), compilation_flag.to_str()); - assert_map_contains!( - outputs, - ( - "obj", - ArtifactDescriptor { - path: "foo.o".into(), - optional: false - } - ) - ); - assert!(preprocessor_args.is_empty()); - assert_eq!(ovec!["-fPIC"], common_args); - assert_eq!(ovec!["-arch", "arm64", "-arch", "i386"], arch_args); - assert!(!msvc_show_includes); + let args = stringvec![ + "-fPIC", "-arch", "arm64", "-arch", "i386", "-o", "foo.o", "-c", "foo.cpp" + ]; + let ParsedArguments { + input, + language, + compilation_flag, + outputs, + preprocessor_args, + msvc_show_includes, + common_args, + arch_args, + .. + } = match parse_arguments_(args, false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.cpp"), input.to_str()); + assert_eq!(Language::Cxx, language); + assert_eq!(Some("-c"), compilation_flag.to_str()); + assert_map_contains!( + outputs, + ( + "obj", + ArtifactDescriptor { + path: "foo.o".into(), + optional: false + } + ) + ); + assert!(preprocessor_args.is_empty()); + assert_eq!(ovec!["-fPIC"], common_args); + assert_eq!(ovec!["-arch", "arm64", "-arch", "i386"], arch_args); + assert!(!msvc_show_includes); + }); } #[test] diff --git a/src/util.rs b/src/util.rs index 737cec096a..ee2d75128e 100644 --- a/src/util.rs +++ b/src/util.rs @@ -13,11 +13,11 @@ // limitations under the License. use crate::mock_command::{CommandChild, RunCommand}; -use ar::Archive; use blake3::Hasher as blake3_Hasher; use byteorder::{BigEndian, ByteOrder}; use fs::File; use fs_err as fs; +use object::{macho, read::archive::ArchiveFile, read::macho::FatArch}; use serde::Serialize; use std::ffi::{OsStr, OsString}; use std::hash::Hasher; @@ -145,13 +145,24 @@ pub async fn hash_all_archives( let path = path.clone(); pool.spawn_blocking(move || -> Result { let mut m = Digest::new(); - let reader = File::open(&path) + let archive_file = File::open(&path) .with_context(|| format!("Failed to open file for hashing: {:?}", path))?; - let mut archive = Archive::new(reader); - while let Some(entry) = archive.next_entry() { - let entry = entry?; - m.update(entry.header().identifier()); - update_from_reader(&mut m, entry)?; + let archive_mmap = + unsafe { memmap2::MmapOptions::new().map_copy_read_only(&archive_file)? }; + + match macho::FatHeader::parse(&*archive_mmap) { + Ok(h) if h.magic.get(object::endian::BigEndian) == macho::FAT_MAGIC => { + for arch in macho::FatHeader::parse_arch32(&*archive_mmap)? { + hash_regular_archive(&mut m, arch.data(&*archive_mmap)?)?; + } + } + Ok(h) if h.magic.get(object::endian::BigEndian) == macho::FAT_MAGIC_64 => { + for arch in macho::FatHeader::parse_arch64(&*archive_mmap)? { + hash_regular_archive(&mut m, arch.data(&*archive_mmap)?)?; + } + } + // Not a FatHeader at all, regular archive. + _ => hash_regular_archive(&mut m, &archive_mmap)?, } Ok(m.finish()) }) @@ -170,15 +181,12 @@ pub async fn hash_all_archives( Ok(hashes.into_iter().map(|res| res.unwrap()).collect()) } -/// Update the digest `m` with all data from `reader`. -fn update_from_reader(m: &mut Digest, mut reader: R) -> Result<()> { - loop { - let mut buffer = [0; 1024]; - let count = reader.read(&mut buffer[..])?; - if count == 0 { - break; - } - m.update(&buffer[..count]); +fn hash_regular_archive(m: &mut Digest, data: &[u8]) -> Result<()> { + let archive = ArchiveFile::parse(data)?; + for entry in archive.members() { + let entry = entry?; + m.update(entry.name()); + m.update(entry.data(data)?); } Ok(()) }