From 18f126d3c29156987f60700d02d623f79c43e635 Mon Sep 17 00:00:00 2001 From: Urgau Date: Tue, 16 Jul 2024 13:37:20 +0200 Subject: [PATCH 1/3] Change stable hashing algorithm to Blake2s 256-bits --- Cargo.lock | 22 ++++++++++++-- compiler/rustc_data_structures/Cargo.toml | 2 +- .../rustc_data_structures/src/fingerprint.rs | 11 +++++-- compiler/rustc_data_structures/src/hashes.rs | 29 ++++++++++++++++--- .../src/stable_hasher.rs | 4 +-- src/tools/tidy/src/deps.rs | 3 ++ 6 files changed, 60 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cafc623c185a6..e3370621d8c73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -362,6 +362,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -1131,6 +1140,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -3518,8 +3528,10 @@ checksum = "5be1bdc7edf596692617627bbfeaba522131b18e06ca4df2b6b689e3c5d5ce84" [[package]] name = "rustc-stable-hash" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c9f15eec8235d7cb775ee6f81891db79b98fd54ba1ad8fae565b88ef1ae4e2" +source = "git+https://github.com/Urgau/rustc-stable-hash.git?rev=5dbe93b#5dbe93b5959372a64f6f7ea60382fe35be55dcb3" +dependencies = [ + "blake2", +] [[package]] name = "rustc-std-workspace-alloc" @@ -5408,6 +5420,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "suggest-tests" version = "0.1.0" diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index e5e733439ea05..b38a504968853 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -15,7 +15,7 @@ jobserver_crate = { version = "0.1.28", package = "jobserver" } measureme = "11" rustc-hash = "1.1.0" rustc-rayon = { version = "0.5.0", optional = true } -rustc-stable-hash = { version = "0.1.0", features = ["nightly"] } +rustc-stable-hash = { git = "https://github.com/Urgau/rustc-stable-hash.git", rev = "5dbe93b", features = ["blake2", "nightly"] } rustc_arena = { path = "../rustc_arena" } rustc_graphviz = { path = "../rustc_graphviz" } rustc_index = { path = "../rustc_index", package = "rustc_index" } diff --git a/compiler/rustc_data_structures/src/fingerprint.rs b/compiler/rustc_data_structures/src/fingerprint.rs index 30e3d6aa86ce9..020341f29ca32 100644 --- a/compiler/rustc_data_structures/src/fingerprint.rs +++ b/compiler/rustc_data_structures/src/fingerprint.rs @@ -158,8 +158,15 @@ impl FromStableHash for Fingerprint { type Hash = StableHasherHash; #[inline] - fn from(StableHasherHash([_0, _1]): Self::Hash) -> Self { - Fingerprint(_0, _1) + fn from(StableHasherHash(bytes): Self::Hash) -> Self { + let p0 = u64::from_le_bytes(bytes[0..8].try_into().unwrap()); + let p1 = u64::from_le_bytes(bytes[8..16].try_into().unwrap()); + let p2 = u64::from_le_bytes(bytes[16..24].try_into().unwrap()); + let p3 = u64::from_le_bytes(bytes[24..32].try_into().unwrap()); + + // See https://stackoverflow.com/a/27952689 on why this function is + // implemented this way. + Fingerprint(p0.wrapping_mul(3).wrapping_add(p1), p2.wrapping_mul(3).wrapping_add(p3)) } } diff --git a/compiler/rustc_data_structures/src/hashes.rs b/compiler/rustc_data_structures/src/hashes.rs index ef5d2e845ef07..0fefbf80c1d80 100644 --- a/compiler/rustc_data_structures/src/hashes.rs +++ b/compiler/rustc_data_structures/src/hashes.rs @@ -60,8 +60,19 @@ impl FromStableHash for Hash64 { type Hash = StableHasherHash; #[inline] - fn from(StableHasherHash([_0, __1]): Self::Hash) -> Self { - Self { inner: _0 } + fn from(StableHasherHash(bytes): Self::Hash) -> Self { + let p0 = u64::from_le_bytes(bytes[0..8].try_into().unwrap()); + let p1 = u64::from_le_bytes(bytes[8..16].try_into().unwrap()); + let p2 = u64::from_le_bytes(bytes[16..24].try_into().unwrap()); + let p3 = u64::from_le_bytes(bytes[24..32].try_into().unwrap()); + + // See https://stackoverflow.com/a/27952689 on why this function is + // implemented this way. + let m0 = p0.wrapping_mul(3).wrapping_add(p1); + let m1 = p2.wrapping_mul(3).wrapping_add(p3); + let h = m0.wrapping_mul(3).wrapping_add(m1); + + Self { inner: h } } } @@ -127,8 +138,18 @@ impl FromStableHash for Hash128 { type Hash = StableHasherHash; #[inline] - fn from(StableHasherHash([_0, _1]): Self::Hash) -> Self { - Self { inner: u128::from(_0) | (u128::from(_1) << 64) } + fn from(StableHasherHash(bytes): Self::Hash) -> Self { + let p0 = u64::from_le_bytes(bytes[0..8].try_into().unwrap()); + let p1 = u64::from_le_bytes(bytes[8..16].try_into().unwrap()); + let p2 = u64::from_le_bytes(bytes[16..24].try_into().unwrap()); + let p3 = u64::from_le_bytes(bytes[24..32].try_into().unwrap()); + + // See https://stackoverflow.com/a/27952689 on why this function is + // implemented this way. + let upper = p0.wrapping_mul(3).wrapping_add(p1); + let lower = p2.wrapping_mul(3).wrapping_add(p3); + + Self { inner: u128::from(lower) | (u128::from(upper) << 64) } } } diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs index 83883eeba9ca0..0ae62475d4eae 100644 --- a/compiler/rustc_data_structures/src/stable_hasher.rs +++ b/compiler/rustc_data_structures/src/stable_hasher.rs @@ -11,9 +11,9 @@ mod tests; pub use crate::hashes::{Hash128, Hash64}; +pub use rustc_stable_hash::hashers::Blake2s256Hash as StableHasherHash; +pub use rustc_stable_hash::hashers::StableBlake2sHasher256 as StableHasher; pub use rustc_stable_hash::FromStableHash; -pub use rustc_stable_hash::SipHasher128Hash as StableHasherHash; -pub use rustc_stable_hash::StableSipHasher128 as StableHasher; /// Something that implements `HashStable` can be hashed in a way that is /// stable across multiple compilation sessions. diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 3c72fae0881e9..8866fa86d4304 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -94,6 +94,7 @@ const EXCEPTIONS: ExceptionList = &[ ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0 // cargo/... (because of serde) ("self_cell", "Apache-2.0"), // rustc (fluent translations) ("snap", "BSD-3-Clause"), // rustc + ("subtle", "BSD-3-Clause"), // rustc blake2 deps ("wasm-encoder", "Apache-2.0 WITH LLVM-exception"), // rustc ("wasmparser", "Apache-2.0 WITH LLVM-exception"), // rustc // tidy-alphabetical-end @@ -235,6 +236,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "arrayvec", "autocfg", "bitflags", + "blake2", "block-buffer", "byteorder", // via ruzstd in object in thorin-dwp "cc", @@ -376,6 +378,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "stacker", "static_assertions", "strsim", + "subtle", "syn", "synstructure", "tempfile", From 9b14ce558b2039e84279166d8a22ab4e6b337956 Mon Sep 17 00:00:00 2001 From: Urgau Date: Tue, 16 Jul 2024 13:43:28 +0200 Subject: [PATCH 2/3] [DO NOT MERGE] Allow git version of rustc-stable-hash --- src/tools/tidy/src/extdeps.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tools/tidy/src/extdeps.rs b/src/tools/tidy/src/extdeps.rs index 8bb80f1171184..d4ae679d67ffc 100644 --- a/src/tools/tidy/src/extdeps.rs +++ b/src/tools/tidy/src/extdeps.rs @@ -9,6 +9,8 @@ const ALLOWED_SOURCES: &[&str] = &[ r#""registry+https://github.com/rust-lang/crates.io-index""#, // This is `rust_team_data` used by `site` in src/tools/rustc-perf, r#""git+https://github.com/rust-lang/team#a5260e76d3aa894c64c56e6ddc8545b9a98043ec""#, + // WIP Blake2 in rustc-stable-hash + r#""git+https://github.com/Urgau/rustc-stable-hash.git?rev=5dbe93b#5dbe93b5959372a64f6f7ea60382fe35be55dcb3""#, ]; /// Checks for external package sources. `root` is the path to the directory that contains the From faed860b3ac9bba42ccaa3d2e440a31e66e98f1b Mon Sep 17 00:00:00 2001 From: Urgau Date: Wed, 17 Jul 2024 14:42:00 +0200 Subject: [PATCH 3/3] let's try blake2s_simd --- Cargo.lock | 33 ++++++++++++++--------- compiler/rustc_data_structures/Cargo.toml | 2 +- src/tools/tidy/src/deps.rs | 8 +++--- src/tools/tidy/src/extdeps.rs | 2 +- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3370621d8c73..a246eb0650f39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -243,6 +243,12 @@ dependencies = [ "object 0.32.2", ] +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + [[package]] name = "arrayvec" version = "0.7.4" @@ -363,12 +369,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] -name = "blake2" -version = "0.10.6" +name = "blake2s_simd" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "94230421e395b9920d23df13ea5d77a20e1725331f90fbbf6df6040b33f756ae" dependencies = [ - "digest", + "arrayref", + "arrayvec", + "constant_time_eq", ] [[package]] @@ -857,6 +865,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + [[package]] name = "core" version = "0.0.0" @@ -1140,7 +1154,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -3528,9 +3541,9 @@ checksum = "5be1bdc7edf596692617627bbfeaba522131b18e06ca4df2b6b689e3c5d5ce84" [[package]] name = "rustc-stable-hash" version = "0.1.0" -source = "git+https://github.com/Urgau/rustc-stable-hash.git?rev=5dbe93b#5dbe93b5959372a64f6f7ea60382fe35be55dcb3" +source = "git+https://github.com/Urgau/rustc-stable-hash.git?rev=4f8c5a9#4f8c5a993536388a22fba12d0be76837dcd52017" dependencies = [ - "blake2", + "blake2s_simd", ] [[package]] @@ -5420,12 +5433,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "suggest-tests" version = "0.1.0" diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index b38a504968853..fae1d471ecae2 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -15,7 +15,7 @@ jobserver_crate = { version = "0.1.28", package = "jobserver" } measureme = "11" rustc-hash = "1.1.0" rustc-rayon = { version = "0.5.0", optional = true } -rustc-stable-hash = { git = "https://github.com/Urgau/rustc-stable-hash.git", rev = "5dbe93b", features = ["blake2", "nightly"] } +rustc-stable-hash = { git = "https://github.com/Urgau/rustc-stable-hash.git", rev = "4f8c5a9", features = ["blake2", "nightly"] } rustc_arena = { path = "../rustc_arena" } rustc_graphviz = { path = "../rustc_graphviz" } rustc_index = { path = "../rustc_index", package = "rustc_index" } diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 8866fa86d4304..3cd05f508cb1c 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -21,6 +21,7 @@ const LICENSES: &[&str] = &[ "Apache-2.0", "Apache-2.0/MIT", "BSD-2-Clause OR Apache-2.0 OR MIT", // zerocopy + "CC0-1.0 OR MIT-0 OR Apache-2.0", // contant_time_eq "ISC", "MIT / Apache-2.0", "MIT OR Apache-2.0 OR LGPL-2.1-or-later", // r-efi, r-efi-alloc @@ -83,6 +84,7 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>) const EXCEPTIONS: ExceptionList = &[ // tidy-alphabetical-start ("ar_archive_writer", "Apache-2.0 WITH LLVM-exception"), // rustc + ("arrayref", "BSD-2-Clause"), // rustc ("colored", "MPL-2.0"), // rustfmt ("dissimilar", "Apache-2.0"), // rustdoc, rustc_lexer (few tests) via expect-test, (dev deps) ("fluent-langneg", "Apache-2.0"), // rustc (fluent translations) @@ -94,7 +96,6 @@ const EXCEPTIONS: ExceptionList = &[ ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0 // cargo/... (because of serde) ("self_cell", "Apache-2.0"), // rustc (fluent translations) ("snap", "BSD-3-Clause"), // rustc - ("subtle", "BSD-3-Clause"), // rustc blake2 deps ("wasm-encoder", "Apache-2.0 WITH LLVM-exception"), // rustc ("wasmparser", "Apache-2.0 WITH LLVM-exception"), // rustc // tidy-alphabetical-end @@ -233,16 +234,18 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "annotate-snippets", "anstyle", "ar_archive_writer", + "arrayref", "arrayvec", "autocfg", "bitflags", - "blake2", + "blake2s_simd", "block-buffer", "byteorder", // via ruzstd in object in thorin-dwp "cc", "cfg-if", "cfg_aliases", "compiler_builtins", + "constant_time_eq", "cpufeatures", "crc32fast", "crossbeam-channel", @@ -378,7 +381,6 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "stacker", "static_assertions", "strsim", - "subtle", "syn", "synstructure", "tempfile", diff --git a/src/tools/tidy/src/extdeps.rs b/src/tools/tidy/src/extdeps.rs index d4ae679d67ffc..fdaa25b3ba17e 100644 --- a/src/tools/tidy/src/extdeps.rs +++ b/src/tools/tidy/src/extdeps.rs @@ -10,7 +10,7 @@ const ALLOWED_SOURCES: &[&str] = &[ // This is `rust_team_data` used by `site` in src/tools/rustc-perf, r#""git+https://github.com/rust-lang/team#a5260e76d3aa894c64c56e6ddc8545b9a98043ec""#, // WIP Blake2 in rustc-stable-hash - r#""git+https://github.com/Urgau/rustc-stable-hash.git?rev=5dbe93b#5dbe93b5959372a64f6f7ea60382fe35be55dcb3""#, + r#""git+https://github.com/Urgau/rustc-stable-hash.git?rev=4f8c5a9#4f8c5a993536388a22fba12d0be76837dcd52017""#, ]; /// Checks for external package sources. `root` is the path to the directory that contains the