From b636fb8ee4460360bcf3e4b1029b4b632a52e958 Mon Sep 17 00:00:00 2001 From: "pinkforest(she/her)" <36498018+pinkforest@users.noreply.github.com> Date: Sun, 8 Sep 2024 15:13:36 +1000 Subject: [PATCH] Make AVX512IFMA opt-in backend (#695) * Make AVX512IFMA opt-in backend * Updated README to have backend info * Added entry to changelog --------- Co-authored-by: Michael Rosenberg --- curve25519-dalek/CHANGELOG.md | 4 ++ curve25519-dalek/Cargo.toml | 2 +- curve25519-dalek/README.md | 19 +++--- curve25519-dalek/build.rs | 60 +++++++++++++------ curve25519-dalek/src/backend/mod.rs | 20 +++---- curve25519-dalek/src/backend/vector/mod.rs | 2 +- .../backend/vector/scalar_mul/pippenger.rs | 5 +- .../vector/scalar_mul/precomputed_straus.rs | 5 +- .../src/backend/vector/scalar_mul/straus.rs | 5 +- .../vector/scalar_mul/variable_base.rs | 5 +- .../vector/scalar_mul/vartime_double_base.rs | 5 +- 11 files changed, 87 insertions(+), 45 deletions(-) diff --git a/curve25519-dalek/CHANGELOG.md b/curve25519-dalek/CHANGELOG.md index 939d3bfba..619aa32ca 100644 --- a/curve25519-dalek/CHANGELOG.md +++ b/curve25519-dalek/CHANGELOG.md @@ -3,6 +3,10 @@ Entries are listed in reverse chronological order per undeprecated major series. +## Unreleased + +* Move AVX-512 backend selection logic to a separate CFG flag that requires nightly + ## 4.x series ### 4.1.3 diff --git a/curve25519-dalek/Cargo.toml b/curve25519-dalek/Cargo.toml index c4fb28a38..fc35264bb 100644 --- a/curve25519-dalek/Cargo.toml +++ b/curve25519-dalek/Cargo.toml @@ -76,7 +76,7 @@ curve25519-dalek-derive = { version = "0.1", path = "../curve25519-dalek-derive" level = "warn" check-cfg = [ 'cfg(allow_unused_unsafe)', - 'cfg(curve25519_dalek_backend, values("fiat", "serial", "simd"))', + 'cfg(curve25519_dalek_backend, values("fiat", "serial", "simd", "unstable_avx512"))', 'cfg(curve25519_dalek_diagnostics, values("build"))', 'cfg(curve25519_dalek_bits, values("32", "64"))', 'cfg(nightly)', diff --git a/curve25519-dalek/README.md b/curve25519-dalek/README.md index c918d6959..e85163636 100644 --- a/curve25519-dalek/README.md +++ b/curve25519-dalek/README.md @@ -90,11 +90,12 @@ This release also does a lot of dependency updates and relaxations to unblock up Curve arithmetic is implemented and used by one of the following backends: -| Backend | Selection | Implementation | Bits / Word sizes | -| :--- | :--- | :--- | :--- | -| `serial` | Automatic | An optimized, non-parllel implementation | `32` and `64` | -| `fiat` | Manual | Formally verified field arithmetic from [fiat-crypto] | `32` and `64` | -| `simd` | Automatic | Intel AVX2 / AVX512 IFMA accelerated backend | `64` only | +| Backend | Selection | Implementation | Bits / Word sizes | +| :--- | :--- | :--- | :--- | +| `serial` | Automatic | An optimized, non-parllel implementation | `32` and `64` | +| `fiat` | Manual | Formally verified field arithmetic from [fiat-crypto] | `32` and `64` | +| `simd` | Automatic | Intel AVX2 accelerated backend | `64` only | +| `unstable_avx512` | Manual | Intel AVX512 IFMA accelerated backend (requires nightly) | `64` only | At runtime, `curve25519-dalek` selects an arithmetic backend from the set of backends it was compiled to support. For Intel x86-64 targets, unless otherwise specified, it will build itself with `simd` support, and default to `serial` at runtime if the appropriate CPU features aren't detected. See [SIMD backend] for more details. @@ -148,16 +149,16 @@ $ cargo build --target i686-unknown-linux-gnu ## SIMD backend -The specific SIMD backend (AVX512 / AVX2 / `serial` default) is selected automatically at runtime, depending on the currently available CPU features, and whether Rust nightly is being used for compilation. The precise conditions are specified below. +When the `simd` backend is selected, the AVX2 or `serial` implementation is selected automatically at runtime, depending on the currently available CPU features. Similarly, when the `unstable_avx512` backend is selected, the AVX512 implementation is selected automatically at runtime if available, or else selection falls through to the aforementioned `simd` backend logic. For a given CPU feature, you can also specify an appropriate `-C target_feature` to build a binary which assumes the required SIMD instructions are always available. Don't do this if you don't have a good reason. | Backend | `RUSTFLAGS` | Requires nightly? | | :--- | :--- | :--- | -| avx2 | `-C target_feature=+avx2` | no | -| avx512 | `-C target_feature=+avx512ifma,+avx512vl` | yes | +| AVX2 | `-C target_feature=+avx2` | no | +| AVX512 | `-C target_feature=+avx512ifma,+avx512vl` | yes | -If compiled on a non-nightly compiler, `curve25519-dalek` will not include AVX512 code, and therefore will never select it at runtime. +To reiterate, the `simd` backend will NOT use AVX512 code under any circumstance. The only way to enable AVX512 currently is to select the `unstable_avx512` backend and use a nightly compiler. # Documentation diff --git a/curve25519-dalek/build.rs b/curve25519-dalek/build.rs index 97fa28524..1b0f618a6 100644 --- a/curve25519-dalek/build.rs +++ b/curve25519-dalek/build.rs @@ -35,13 +35,16 @@ fn main() { println!("cargo:rustc-cfg=curve25519_dalek_bits=\"{curve25519_dalek_bits}\""); - if rustc_version::version_meta() + let nightly = if rustc_version::version_meta() .expect("failed to detect rustc version") .channel == rustc_version::Channel::Nightly { println!("cargo:rustc-cfg=nightly"); - } + true + } else { + false + }; let rustc_version = rustc_version::version().expect("failed to detect rustc version"); if rustc_version.major == 1 && rustc_version.minor <= 64 { @@ -51,25 +54,44 @@ fn main() { } // Backend overrides / defaults - let curve25519_dalek_backend = - match std::env::var("CARGO_CFG_CURVE25519_DALEK_BACKEND").as_deref() { - Ok("fiat") => "fiat", - Ok("serial") => "serial", - Ok("simd") => { - // simd can only be enabled on x86_64 & 64bit target_pointer_width - match is_capable_simd(&target_arch, curve25519_dalek_bits) { - true => "simd", - // If override is not possible this must result to compile error - // See: issues/532 - false => panic!("Could not override curve25519_dalek_backend to simd"), + let curve25519_dalek_backend = match std::env::var("CARGO_CFG_CURVE25519_DALEK_BACKEND") + .as_deref() + { + Ok("fiat") => "fiat", + Ok("serial") => "serial", + Ok("simd") => { + // simd can only be enabled on x86_64 & 64bit target_pointer_width + match is_capable_simd(&target_arch, curve25519_dalek_bits) { + true => "simd", + // If override is not possible this must result to compile error + // See: issues/532 + false => panic!("Could not override curve25519_dalek_backend to simd"), + } + } + Ok("unstable_avx512") if nightly => { + // simd can only be enabled on x86_64 & 64bit target_pointer_width + match is_capable_simd(&target_arch, curve25519_dalek_bits) { + true => { + // In addition enable Avx2 fallback through simd stable backend + // NOTE: Compiler permits duplicate / multi value on the same key + println!("cargo:rustc-cfg=curve25519_dalek_backend=\"simd\""); + + "unstable_avx512" } + // If override is not possible this must result to compile error + // See: issues/532 + false => panic!("Could not override curve25519_dalek_backend to unstable_avx512"), } - // default between serial / simd (if potentially capable) - _ => match is_capable_simd(&target_arch, curve25519_dalek_bits) { - true => "simd", - false => "serial", - }, - }; + } + Ok("unstable_avx512") if !nightly => { + panic!("Could not override curve25519_dalek_backend to unstable_avx512, as this is nigthly only"); + } + // default between serial / simd (if potentially capable) + _ => match is_capable_simd(&target_arch, curve25519_dalek_bits) { + true => "simd", + false => "serial", + }, + }; println!("cargo:rustc-cfg=curve25519_dalek_backend=\"{curve25519_dalek_backend}\""); } diff --git a/curve25519-dalek/src/backend/mod.rs b/curve25519-dalek/src/backend/mod.rs index 9ad1dd3de..c5fab97f5 100644 --- a/curve25519-dalek/src/backend/mod.rs +++ b/curve25519-dalek/src/backend/mod.rs @@ -46,14 +46,14 @@ pub mod vector; enum BackendKind { #[cfg(curve25519_dalek_backend = "simd")] Avx2, - #[cfg(all(curve25519_dalek_backend = "simd", nightly))] + #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] Avx512, Serial, } #[inline] fn get_selected_backend() -> BackendKind { - #[cfg(all(curve25519_dalek_backend = "simd", nightly))] + #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] { cpufeatures::new!(cpuid_avx512, "avx512ifma", "avx512vl"); let token_avx512: cpuid_avx512::InitToken = cpuid_avx512::init(); @@ -88,7 +88,7 @@ where #[cfg(curve25519_dalek_backend = "simd")] BackendKind::Avx2 => vector::scalar_mul::pippenger::spec_avx2::Pippenger::optional_multiscalar_mul::(scalars, points), - #[cfg(all(curve25519_dalek_backend = "simd", nightly))] + #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] BackendKind::Avx512 => vector::scalar_mul::pippenger::spec_avx512ifma_avx512vl::Pippenger::optional_multiscalar_mul::(scalars, points), BackendKind::Serial => @@ -100,7 +100,7 @@ where pub(crate) enum VartimePrecomputedStraus { #[cfg(curve25519_dalek_backend = "simd")] Avx2(vector::scalar_mul::precomputed_straus::spec_avx2::VartimePrecomputedStraus), - #[cfg(all(curve25519_dalek_backend = "simd", nightly))] + #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] Avx512ifma( vector::scalar_mul::precomputed_straus::spec_avx512ifma_avx512vl::VartimePrecomputedStraus, ), @@ -120,7 +120,7 @@ impl VartimePrecomputedStraus { #[cfg(curve25519_dalek_backend = "simd")] BackendKind::Avx2 => VartimePrecomputedStraus::Avx2(vector::scalar_mul::precomputed_straus::spec_avx2::VartimePrecomputedStraus::new(static_points)), - #[cfg(all(curve25519_dalek_backend = "simd", nightly))] + #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] BackendKind::Avx512 => VartimePrecomputedStraus::Avx512ifma(vector::scalar_mul::precomputed_straus::spec_avx512ifma_avx512vl::VartimePrecomputedStraus::new(static_points)), BackendKind::Serial => @@ -150,7 +150,7 @@ impl VartimePrecomputedStraus { dynamic_scalars, dynamic_points, ), - #[cfg(all(curve25519_dalek_backend = "simd", nightly))] + #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] VartimePrecomputedStraus::Avx512ifma(inner) => inner.optional_mixed_multiscalar_mul( static_scalars, dynamic_scalars, @@ -181,7 +181,7 @@ where BackendKind::Avx2 => { vector::scalar_mul::straus::spec_avx2::Straus::multiscalar_mul::(scalars, points) } - #[cfg(all(curve25519_dalek_backend = "simd", nightly))] + #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] BackendKind::Avx512 => { vector::scalar_mul::straus::spec_avx512ifma_avx512vl::Straus::multiscalar_mul::( scalars, points, @@ -210,7 +210,7 @@ where scalars, points, ) } - #[cfg(all(curve25519_dalek_backend = "simd", nightly))] + #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] BackendKind::Avx512 => { vector::scalar_mul::straus::spec_avx512ifma_avx512vl::Straus::optional_multiscalar_mul::< I, @@ -228,7 +228,7 @@ pub fn variable_base_mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint match get_selected_backend() { #[cfg(curve25519_dalek_backend = "simd")] BackendKind::Avx2 => vector::scalar_mul::variable_base::spec_avx2::mul(point, scalar), - #[cfg(all(curve25519_dalek_backend = "simd", nightly))] + #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] BackendKind::Avx512 => { vector::scalar_mul::variable_base::spec_avx512ifma_avx512vl::mul(point, scalar) } @@ -242,7 +242,7 @@ pub fn vartime_double_base_mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> Edwa match get_selected_backend() { #[cfg(curve25519_dalek_backend = "simd")] BackendKind::Avx2 => vector::scalar_mul::vartime_double_base::spec_avx2::mul(a, A, b), - #[cfg(all(curve25519_dalek_backend = "simd", nightly))] + #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] BackendKind::Avx512 => { vector::scalar_mul::vartime_double_base::spec_avx512ifma_avx512vl::mul(a, A, b) } diff --git a/curve25519-dalek/src/backend/vector/mod.rs b/curve25519-dalek/src/backend/vector/mod.rs index 2839dca45..3a9104412 100644 --- a/curve25519-dalek/src/backend/vector/mod.rs +++ b/curve25519-dalek/src/backend/vector/mod.rs @@ -16,7 +16,7 @@ pub mod packed_simd; pub mod avx2; -#[cfg(nightly)] +#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))] pub mod ifma; pub mod scalar_mul; diff --git a/curve25519-dalek/src/backend/vector/scalar_mul/pippenger.rs b/curve25519-dalek/src/backend/vector/scalar_mul/pippenger.rs index 1376c4eab..ed89cdefe 100644 --- a/curve25519-dalek/src/backend/vector/scalar_mul/pippenger.rs +++ b/curve25519-dalek/src/backend/vector/scalar_mul/pippenger.rs @@ -11,7 +11,10 @@ #[curve25519_dalek_derive::unsafe_target_feature_specialize( "avx2", - conditional("avx512ifma,avx512vl", nightly) + conditional( + "avx512ifma,avx512vl", + all(curve25519_dalek_backend = "unstable_avx512", nightly) + ) )] pub mod spec { diff --git a/curve25519-dalek/src/backend/vector/scalar_mul/precomputed_straus.rs b/curve25519-dalek/src/backend/vector/scalar_mul/precomputed_straus.rs index 1f16ab3e1..6038d18c7 100644 --- a/curve25519-dalek/src/backend/vector/scalar_mul/precomputed_straus.rs +++ b/curve25519-dalek/src/backend/vector/scalar_mul/precomputed_straus.rs @@ -13,7 +13,10 @@ #[curve25519_dalek_derive::unsafe_target_feature_specialize( "avx2", - conditional("avx512ifma,avx512vl", nightly) + conditional( + "avx512ifma,avx512vl", + all(curve25519_dalek_backend = "unstable_avx512", nightly) + ) )] pub mod spec { diff --git a/curve25519-dalek/src/backend/vector/scalar_mul/straus.rs b/curve25519-dalek/src/backend/vector/scalar_mul/straus.rs index 413e6fd9a..70e4949db 100644 --- a/curve25519-dalek/src/backend/vector/scalar_mul/straus.rs +++ b/curve25519-dalek/src/backend/vector/scalar_mul/straus.rs @@ -13,7 +13,10 @@ #[curve25519_dalek_derive::unsafe_target_feature_specialize( "avx2", - conditional("avx512ifma,avx512vl", nightly) + conditional( + "avx512ifma,avx512vl", + all(curve25519_dalek_backend = "unstable_avx512", nightly) + ) )] pub mod spec { diff --git a/curve25519-dalek/src/backend/vector/scalar_mul/variable_base.rs b/curve25519-dalek/src/backend/vector/scalar_mul/variable_base.rs index 9f924f286..585945209 100644 --- a/curve25519-dalek/src/backend/vector/scalar_mul/variable_base.rs +++ b/curve25519-dalek/src/backend/vector/scalar_mul/variable_base.rs @@ -2,7 +2,10 @@ #[curve25519_dalek_derive::unsafe_target_feature_specialize( "avx2", - conditional("avx512ifma,avx512vl", nightly) + conditional( + "avx512ifma,avx512vl", + all(curve25519_dalek_backend = "unstable_avx512", nightly) + ) )] pub mod spec { diff --git a/curve25519-dalek/src/backend/vector/scalar_mul/vartime_double_base.rs b/curve25519-dalek/src/backend/vector/scalar_mul/vartime_double_base.rs index ea2af8ad4..64254e248 100644 --- a/curve25519-dalek/src/backend/vector/scalar_mul/vartime_double_base.rs +++ b/curve25519-dalek/src/backend/vector/scalar_mul/vartime_double_base.rs @@ -13,7 +13,10 @@ #[curve25519_dalek_derive::unsafe_target_feature_specialize( "avx2", - conditional("avx512ifma,avx512vl", nightly) + conditional( + "avx512ifma,avx512vl", + all(curve25519_dalek_backend = "unstable_avx512", nightly) + ) )] pub mod spec {