implement carryless_mul

Author: folkertdev

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -396,6 +396,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             | sym::bitreverse
             | sym::saturating_add
             | sym::saturating_sub
+            | sym::carryless_mul
             | sym::unchecked_funnel_shl
             | sym::unchecked_funnel_shr => {
                 let ty = args[0].layout.ty;
@@ -438,6 +439,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     sym::bitreverse => {
                         self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
                     }
+                    sym::carryless_mul => {
+                        let lhs = args[0].immediate();
+                        let rhs = args[1].immediate();
+                        self.call_intrinsic("llvm.clmul", &[llty], &[lhs, rhs])
+                    }
                     sym::unchecked_funnel_shl | sym::unchecked_funnel_shr => {
                         let is_left = name == sym::unchecked_funnel_shl;
                         let lhs = args[0].immediate();
@@ -2787,6 +2793,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             sym::simd_cttz => "llvm.cttz",
             sym::simd_funnel_shl => "llvm.fshl",
             sym::simd_funnel_shr => "llvm.fshr",
+            sym::simd_carryless_mul => "llvm.clmul",
             _ => unreachable!(),
         };
         let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
@@ -2812,6 +2819,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 &[vec_ty],
                 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
             )),
+            sym::simd_carryless_mul => Ok(bx.call_intrinsic(
+                llvm_intrinsic,
+                &[vec_ty],
+                &[args[0].immediate(), args[1].immediate()],
+            )),
             _ => unreachable!(),
         };
     }

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -82,6 +82,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi
         | sym::bswap
         | sym::caller_location
         | sym::carrying_mul_add
+        | sym::carryless_mul
         | sym::ceilf16
         | sym::ceilf32
         | sym::ceilf64
@@ -564,6 +565,7 @@ pub(crate) fn check_intrinsic_type(
             (1, 0, vec![param(0), param(0)], param(0))
         }
         sym::saturating_add | sym::saturating_sub => (1, 0, vec![param(0), param(0)], param(0)),
+        sym::carryless_mul => (1, 0, vec![param(0), param(0)], param(0)),
         sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
             (1, 0, vec![param(0), param(0)], param(0))
         }
@@ -711,7 +713,8 @@ pub(crate) fn check_intrinsic_type(
         | sym::simd_fmin
         | sym::simd_fmax
         | sym::simd_saturating_add
-        | sym::simd_saturating_sub => (1, 0, vec![param(0), param(0)], param(0)),
+        | sym::simd_saturating_sub
+        | sym::simd_carryless_mul => (1, 0, vec![param(0), param(0)], param(0)),
         sym::simd_arith_offset => (2, 0, vec![param(0), param(1)], param(0)),
         sym::simd_neg
         | sym::simd_bswap

compiler/rustc_span/src/symbol.rs

@@ -642,6 +642,7 @@ symbols! {
         caller_location,
         capture_disjoint_fields,
         carrying_mul_add,
+        carryless_mul,
         catch_unwind,
         cause,
         cdylib,
@@ -2083,6 +2084,7 @@ symbols! {
         simd_bitmask,
         simd_bitreverse,
         simd_bswap,
+        simd_carryless_mul,
         simd_cast,
         simd_cast_ptr,
         simd_ceil,

library/core/src/intrinsics/fallback.rs

@@ -218,3 +218,57 @@ macro_rules! impl_funnel_shifts {
 impl_funnel_shifts! {
     u8, u16, u32, u64, u128, usize
 }
+
+#[rustc_const_unstable(feature = "core_intrinsics_fallbacks", issue = "none")]
+pub const trait CarrylessMul: Copy + 'static {
+    /// See [`super::carryless_mul`]; we just need the trait indirection to handle
+    /// different types since calling intrinsics with generics doesn't work.
+    fn carryless_mul(self, rhs: Self) -> Self;
+}
+
+macro_rules! impl_carryless_mul{
+    ($($type:ident),*) => {$(
+        /// This approach uses a bitmask of the form `0b100010001...0001` to avoid carry spilling.
+        /// When carries do occur, they wind up in a "hole" of zeros and are subsequently masked
+        /// out of the result.
+        #[rustc_const_unstable(feature = "core_intrinsics_fallbacks", issue = "none")]
+        impl const CarrylessMul for $type {
+            #[inline]
+            fn carryless_mul(self, rhs: Self) -> Self {
+                use crate::num::Wrapping;
+
+                // i.e. 0b100010001...0001 in binary.
+                const MASK: u128 = 0x1111_1111_1111_1111_1111_1111_1111_1111;
+
+                let m0 = MASK as $type;
+                let x = self;
+                let y = rhs;
+
+                let m1 = m0 << 1;
+                let m2 = m1 << 1;
+                let m3 = m2 << 1;
+
+                let x0 = Wrapping(x & m0);
+                let x1 = Wrapping(x & m1);
+                let x2 = Wrapping(x & m2);
+                let x3 = Wrapping(x & m3);
+
+                let y0 = Wrapping(y & m0);
+                let y1 = Wrapping(y & m1);
+                let y2 = Wrapping(y & m2);
+                let y3 = Wrapping(y & m3);
+
+                let z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
+                let z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
+                let z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
+                let z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
+
+                (z0.0 & m0) | (z1.0 & m1) | (z2.0 & m2) | (z3.0 & m3)
+            }
+        }
+    )*};
+}
+
+impl_carryless_mul! {
+    u8, u16, u32, u64, u128, usize
+}

library/core/src/intrinsics/mod.rs

@@ -2179,6 +2179,19 @@ pub const unsafe fn unchecked_funnel_shr<T: [const] fallback::FunnelShift>(
     unsafe { a.unchecked_funnel_shr(b, shift) }
 }
 
+/// Carryless multiply.
+///
+/// Safe versions of this intrinsic are available on the integer primitives
+/// via the `carryless_mul` method. For example, [`u32::carryless_mul`].
+#[rustc_intrinsic]
+#[rustc_nounwind]
+#[rustc_const_unstable(feature = "uint_carryless_mul", issue = "152080")]
+#[unstable(feature = "uint_carryless_mul", issue = "152080")]
+#[miri::intrinsic_fallback_is_spec]
+pub const fn carryless_mul<T: [const] fallback::CarrylessMul>(a: T, b: T) -> T {
+    a.carryless_mul(b)
+}
+
 /// This is an implementation detail of [`crate::ptr::read`] and should
 /// not be used anywhere else.  See its comments for why this exists.
 ///

library/core/src/intrinsics/simd.rs

@@ -162,6 +162,18 @@ pub const unsafe fn simd_funnel_shl<T>(a: T, b: T, shift: T) -> T;
 #[rustc_nounwind]
 pub const unsafe fn simd_funnel_shr<T>(a: T, b: T, shift: T) -> T;
 
+/// Compute the carry-less product.
+///
+/// This is similar to long multiplication except that the carry is discarded.
+///
+/// This operation can be used to model multiplication in `GF(2)[X]`, the polynomial
+/// ring over `GF(2)`.
+///
+/// `T` must be a vector of integers.
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub unsafe fn simd_carryless_mul<T>(a: T, b: T) -> T;
+
 /// "And"s vectors elementwise.
 ///
 /// `T` must be a vector of integers.

library/core/src/lib.rs

@@ -188,6 +188,7 @@
 #![feature(trait_alias)]
 #![feature(transparent_unions)]
 #![feature(try_blocks)]
+#![feature(uint_carryless_mul)]
 #![feature(unboxed_closures)]
 #![feature(unsized_fn_params)]
 #![feature(with_negative_coherence)]

library/core/src/num/mod.rs

@@ -458,6 +458,9 @@ impl u8 {
         fsh_op = "0x36",
         fshl_result = "0x8",
         fshr_result = "0x8d",
+        clmul_lhs = "0x12",
+        clmul_rhs = "0x34",
+        clmul_result = "0x28",
         swap_op = "0x12",
         swapped = "0x12",
         reversed = "0x48",
@@ -1095,6 +1098,9 @@ impl u16 {
         fsh_op = "0x2de",
         fshl_result = "0x30",
         fshr_result = "0x302d",
+        clmul_lhs = "0x9012",
+        clmul_rhs = "0xcd34",
+        clmul_result = "0x928",
         swap_op = "0x1234",
         swapped = "0x3412",
         reversed = "0x2c48",
@@ -1145,6 +1151,9 @@ impl u32 {
         fsh_op = "0x2fe78e45",
         fshl_result = "0xb32f",
         fshr_result = "0xb32fe78e",
+        clmul_lhs = "0x56789012",
+        clmul_rhs = "0xf52ecd34",
+        clmul_result = "0x9b980928",
         swap_op = "0x12345678",
         swapped = "0x78563412",
         reversed = "0x1e6a2c48",
@@ -1171,6 +1180,9 @@ impl u64 {
         fsh_op = "0x2fe78e45983acd98",
         fshl_result = "0x6e12fe",
         fshr_result = "0x6e12fe78e45983ac",
+        clmul_lhs = "0x7890123456789012",
+        clmul_rhs = "0xdd358416f52ecd34",
+        clmul_result = "0xa6299579b980928",
         swap_op = "0x1234567890123456",
         swapped = "0x5634129078563412",
         reversed = "0x6a2c48091e6a2c48",
@@ -1197,6 +1209,9 @@ impl u128 {
         fsh_op = "0x2fe78e45983acd98039000008736273",
         fshl_result = "0x4f7602fe",
         fshr_result = "0x4f7602fe78e45983acd9803900000873",
+        clmul_lhs = "0x12345678901234567890123456789012",
+        clmul_rhs = "0x4317e40ab4ddcf05dd358416f52ecd34",
+        clmul_result = "0xb9cf660de35d0c170a6299579b980928",
         swap_op = "0x12345678901234567890123456789012",
         swapped = "0x12907856341290785634129078563412",
         reversed = "0x48091e6a2c48091e6a2c48091e6a2c48",
@@ -1223,9 +1238,12 @@ impl usize {
         rot = 4,
         rot_op = "0xa003",
         rot_result = "0x3a",
-        fsh_op = "0x2fe78e45983acd98039000008736273",
-        fshl_result = "0x4f7602fe",
-        fshr_result = "0x4f7602fe78e45983acd9803900000873",
+        fsh_op = "0x2de",
+        fshl_result = "0x30",
+        fshr_result = "0x302d",
+        clmul_lhs = "0x9012",
+        clmul_rhs = "0xcd34",
+        clmul_result = "0x928",
         swap_op = "0x1234",
         swapped = "0x3412",
         reversed = "0x2c48",
@@ -1253,6 +1271,9 @@ impl usize {
         fsh_op = "0x2fe78e45",
         fshl_result = "0xb32f",
         fshr_result = "0xb32fe78e",
+        clmul_lhs = "0x56789012",
+        clmul_rhs = "0xf52ecd34",
+        clmul_result = "0x9b980928",
         swap_op = "0x12345678",
         swapped = "0x78563412",
         reversed = "0x1e6a2c48",
@@ -1280,6 +1301,9 @@ impl usize {
         fsh_op = "0x2fe78e45983acd98",
         fshl_result = "0x6e12fe",
         fshr_result = "0x6e12fe78e45983ac",
+        clmul_lhs = "0x7890123456789012",
+        clmul_rhs = "0xdd358416f52ecd34",
+        clmul_result = "0xa6299579b980928",
         swap_op = "0x1234567890123456",
         swapped = "0x5634129078563412",
         reversed = "0x6a2c48091e6a2c48",

library/core/src/num/uint_macros.rs

@@ -17,6 +17,9 @@ macro_rules! uint_impl {
         fsh_op = $fsh_op:literal,
         fshl_result = $fshl_result:literal,
         fshr_result = $fshr_result:literal,
+        clmul_lhs = $clmul_rhs:literal,
+        clmul_rhs = $clmul_lhs:literal,
+        clmul_result = $clmul_result:literal,
         swap_op = $swap_op:literal,
         swapped = $swapped:literal,
         reversed = $reversed:literal,
@@ -482,6 +485,32 @@ macro_rules! uint_impl {
             unsafe { intrinsics::unchecked_funnel_shr(self, rhs, n) }
         }
 
+        /// Performs a carry-less multiplication.
+        ///
+        /// This is similar to long multiplication except that the carry is discarded.
+        /// This function wraps, so only the low bits are returned.
+        ///
+        /// This operation can be used to model multiplication in `GF(2)[X]`, the polynomial
+        /// ring over `GF(2)`.
+        ///
+        /// ```
+        /// #![feature(uint_carryless_mul)]
+        ///
+        #[doc = concat!("let a = ", $clmul_lhs, stringify!($SelfT), ";")]
+        #[doc = concat!("let b = ", $clmul_rhs, stringify!($SelfT), ";")]
+        ///
+        #[doc = concat!("assert_eq!(a.carryless_mul(b), ", $clmul_result, ");")]
+        /// ```
+        #[rustc_const_unstable(feature = "uint_carryless_mul", issue = "152080")]
+        #[doc(alias = "clmul")]
+        #[unstable(feature = "uint_carryless_mul", issue = "152080")]
+        #[must_use = "this returns the result of the operation, \
+                      without modifying the original"]
+        #[inline(always)]
+        pub const fn carryless_mul(self, rhs: Self) -> Self {
+            intrinsics::carryless_mul(self, rhs)
+        }
+
         /// Reverses the byte order of the integer.
         ///
         /// # Examples

library/coretests/tests/lib.rs

@@ -120,6 +120,7 @@
 #![feature(try_trait_v2)]
 #![feature(type_info)]
 #![feature(uint_bit_width)]
+#![feature(uint_carryless_mul)]
 #![feature(uint_gather_scatter_bits)]
 #![feature(unsize)]
 #![feature(unwrap_infallible)]