From ac8ea510525ccc365c371fa48ef5a7f99d5ee1df Mon Sep 17 00:00:00 2001 From: Jarrod Petz Date: Wed, 7 Oct 2026 23:34:57 +1100 Subject: [PATCH] Add `_movdir64b`, `_directstoreu_u32` and `_directstoreu_u64` intrinsics Add the intrinsics for the `MOVDIR64B` (64-byte direct store) and `MOVDIRI` (32/64-bit integer direct store) instructions, gated on the `movdir64b` and `movdiri` target features. They link `llvm.x86.movdir64b` and `llvm.x86.directstore32`/`directstore64`, and have `assert_instr` codegen tests and `simd_test` runtime tests. `_directstoreu_u64` lives in `x86_64` since `MOVDIRI` with a 64-bit operand needs 64-bit mode. stdarch-verify maps `MOVDIR64B`'s `void*` operands to `*u8` and `MOVDIRI`'s to the integer type it stores, and missing-x86.md is regenerated. --- crates/core_arch/missing-x86.md | 13 ------ crates/core_arch/src/lib.rs | 2 + crates/core_arch/src/x86/mod.rs | 8 ++++ crates/core_arch/src/x86/movdir64b.rs | 57 ++++++++++++++++++++++++ crates/core_arch/src/x86/movdiri.rs | 51 +++++++++++++++++++++ crates/core_arch/src/x86_64/mod.rs | 4 ++ crates/core_arch/src/x86_64/movdiri.rs | 51 +++++++++++++++++++++ crates/stdarch-verify/tests/x86-intel.rs | 11 ++++- 8 files changed, 182 insertions(+), 15 deletions(-) create mode 100644 crates/core_arch/src/x86/movdir64b.rs create mode 100644 crates/core_arch/src/x86/movdiri.rs create mode 100644 crates/core_arch/src/x86_64/movdiri.rs diff --git a/crates/core_arch/missing-x86.md b/crates/core_arch/missing-x86.md index 637c8d4a86..9e9d3c5514 100644 --- a/crates/core_arch/missing-x86.md +++ b/crates/core_arch/missing-x86.md @@ -96,19 +96,6 @@

-
["MOVDIR64B"]

- - * [ ] [`_movdir64b`](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_movdir64b) -

- - -
["MOVDIRI"]

- - * [ ] [`_directstoreu_u32`](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_directstoreu_u32) - * [ ] [`_directstoreu_u64`](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_directstoreu_u64) -

- -
["PCONFIG"]

* [ ] [`_pconfig_u32`](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_pconfig_u32) diff --git a/crates/core_arch/src/lib.rs b/crates/core_arch/src/lib.rs index 2b7fb15913..0d18174349 100644 --- a/crates/core_arch/src/lib.rs +++ b/crates/core_arch/src/lib.rs @@ -39,6 +39,8 @@ maybe_uninit_as_bytes, movrs_target_feature, clflushopt_target_feature, + movdir64b_target_feature, + movdiri_target_feature, min_adt_const_params )] #![cfg_attr( diff --git a/crates/core_arch/src/x86/mod.rs b/crates/core_arch/src/x86/mod.rs index e6875a628f..4a5d10adc4 100644 --- a/crates/core_arch/src/x86/mod.rs +++ b/crates/core_arch/src/x86/mod.rs @@ -702,6 +702,14 @@ mod clflushopt; #[unstable(feature = "simd_x86_clflushopt", issue = "157096")] pub use self::clflushopt::*; +mod movdir64b; +#[unstable(feature = "simd_x86_movdir64b", issue = "163741")] +pub use self::movdir64b::*; + +mod movdiri; +#[unstable(feature = "simd_x86_movdiri", issue = "163741")] +pub use self::movdiri::*; + #[cfg(test)] use stdarch_test::assert_instr; diff --git a/crates/core_arch/src/x86/movdir64b.rs b/crates/core_arch/src/x86/movdir64b.rs new file mode 100644 index 0000000000..e6325fabf7 --- /dev/null +++ b/crates/core_arch/src/x86/movdir64b.rs @@ -0,0 +1,57 @@ +//! `MOVDIR64B` 64-byte direct store. + +#[cfg(test)] +use stdarch_test::assert_instr; + +#[allow(improper_ctypes)] +unsafe extern "llvm-intrinsic" { + #[link_name = "llvm.x86.movdir64b"] + fn movdir64b(dst: *mut u8, src: *const u8); +} + +/// Moves a 64-byte value from `src` to `dst` using a direct store. +/// +/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_movdir64b) +/// +/// # Safety +/// +/// `src` must be valid for reads of 64 bytes; it does not need to be aligned. `dst` must be valid +/// for writes of 64 bytes and aligned to 64 bytes (`MOVDIR64B` raises a general-protection fault +/// otherwise). +/// +/// # Safety of direct stores +/// +/// Like a non-temporal store, the direct store is weakly ordered with respect to other stores. +/// After using this intrinsic, but before any other access to the memory that this intrinsic +/// mutates, a call to [`_mm_sfence`] must be performed by the thread that used the intrinsic. In +/// particular, functions that call this intrinsic should generally call `_mm_sfence` before they +/// return. +/// +/// See [`_mm_sfence`] for details. +/// +/// [`_mm_sfence`]: crate::arch::x86::_mm_sfence +#[inline] +#[target_feature(enable = "movdir64b")] +#[cfg_attr(test, assert_instr(movdir64b))] +#[unstable(feature = "simd_x86_movdir64b", issue = "163741")] +pub unsafe fn _movdir64b(dst: *mut u8, src: *const u8) { + movdir64b(dst, src); +} + +#[cfg(test)] +mod tests { + use crate::core_arch::x86::*; + use stdarch_test::simd_test; + + #[repr(C, align(64))] + struct Line([u8; 64]); + + #[simd_test(enable = "movdir64b")] + unsafe fn test_movdir64b() { + let src = Line(core::array::from_fn(|i| i as u8)); + let mut dst = Line([0; 64]); + _movdir64b(dst.0.as_mut_ptr(), src.0.as_ptr()); + _mm_sfence(); + assert_eq!(dst.0, src.0); + } +} diff --git a/crates/core_arch/src/x86/movdiri.rs b/crates/core_arch/src/x86/movdiri.rs new file mode 100644 index 0000000000..24287e483f --- /dev/null +++ b/crates/core_arch/src/x86/movdiri.rs @@ -0,0 +1,51 @@ +//! `MOVDIRI` doubleword direct store. + +#[cfg(test)] +use stdarch_test::assert_instr; + +#[allow(improper_ctypes)] +unsafe extern "llvm-intrinsic" { + #[link_name = "llvm.x86.directstore32"] + fn directstore32(dst: *mut u32, val: u32); +} + +/// Stores the 32-bit integer `val` to `dst` using a direct store. +/// +/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_directstoreu_u32) +/// +/// # Safety +/// +/// `dst` must be valid for writes of 4 bytes; it does not need to be aligned. +/// +/// # Safety of direct stores +/// +/// Like a non-temporal store, the direct store is weakly ordered with respect to other stores. +/// After using this intrinsic, but before any other access to the memory that this intrinsic +/// mutates, a call to [`_mm_sfence`] must be performed by the thread that used the intrinsic. In +/// particular, functions that call this intrinsic should generally call `_mm_sfence` before they +/// return. +/// +/// See [`_mm_sfence`] for details. +/// +/// [`_mm_sfence`]: crate::arch::x86::_mm_sfence +#[inline] +#[target_feature(enable = "movdiri")] +#[cfg_attr(test, assert_instr(movdiri))] +#[unstable(feature = "simd_x86_movdiri", issue = "163741")] +pub unsafe fn _directstoreu_u32(dst: *mut u32, val: u32) { + directstore32(dst, val); +} + +#[cfg(test)] +mod tests { + use crate::core_arch::x86::*; + use stdarch_test::simd_test; + + #[simd_test(enable = "movdiri")] + unsafe fn test_directstoreu_u32() { + let mut x = 0_u32; + _directstoreu_u32(&mut x, 0xdead_beef); + _mm_sfence(); + assert_eq!(x, 0xdead_beef); + } +} diff --git a/crates/core_arch/src/x86_64/mod.rs b/crates/core_arch/src/x86_64/mod.rs index 6f1177ad94..b2ca471991 100644 --- a/crates/core_arch/src/x86_64/mod.rs +++ b/crates/core_arch/src/x86_64/mod.rs @@ -119,6 +119,10 @@ mod amx; #[unstable(feature = "x86_amx_intrinsics", issue = "126622")] pub use self::amx::*; +mod movdiri; +#[unstable(feature = "simd_x86_movdiri", issue = "163741")] +pub use self::movdiri::*; + mod movrs; #[unstable(feature = "movrs_target_feature", issue = "137976")] pub use self::movrs::*; diff --git a/crates/core_arch/src/x86_64/movdiri.rs b/crates/core_arch/src/x86_64/movdiri.rs new file mode 100644 index 0000000000..d407c830bf --- /dev/null +++ b/crates/core_arch/src/x86_64/movdiri.rs @@ -0,0 +1,51 @@ +//! `MOVDIRI` quadword direct store. + +#[cfg(test)] +use stdarch_test::assert_instr; + +#[allow(improper_ctypes)] +unsafe extern "llvm-intrinsic" { + #[link_name = "llvm.x86.directstore64"] + fn directstore64(dst: *mut u64, val: u64); +} + +/// Stores the 64-bit integer `val` to `dst` using a direct store. +/// +/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_directstoreu_u64) +/// +/// # Safety +/// +/// `dst` must be valid for writes of 8 bytes; it does not need to be aligned. +/// +/// # Safety of direct stores +/// +/// Like a non-temporal store, the direct store is weakly ordered with respect to other stores. +/// After using this intrinsic, but before any other access to the memory that this intrinsic +/// mutates, a call to [`_mm_sfence`] must be performed by the thread that used the intrinsic. In +/// particular, functions that call this intrinsic should generally call `_mm_sfence` before they +/// return. +/// +/// See [`_mm_sfence`] for details. +/// +/// [`_mm_sfence`]: crate::arch::x86_64::_mm_sfence +#[inline] +#[target_feature(enable = "movdiri")] +#[cfg_attr(test, assert_instr(movdiri))] +#[unstable(feature = "simd_x86_movdiri", issue = "163741")] +pub unsafe fn _directstoreu_u64(dst: *mut u64, val: u64) { + directstore64(dst, val); +} + +#[cfg(test)] +mod tests { + use crate::core_arch::{x86::*, x86_64::*}; + use stdarch_test::simd_test; + + #[simd_test(enable = "movdiri")] + unsafe fn test_directstoreu_u64() { + let mut x = 0_u64; + _directstoreu_u64(&mut x, 0x0123_4567_89ab_cdef); + _mm_sfence(); + assert_eq!(x, 0x0123_4567_89ab_cdef); + } +} diff --git a/crates/stdarch-verify/tests/x86-intel.rs b/crates/stdarch-verify/tests/x86-intel.rs index d2839bb300..809af2ef40 100644 --- a/crates/stdarch-verify/tests/x86-intel.rs +++ b/crates/stdarch-verify/tests/x86-intel.rs @@ -675,15 +675,22 @@ fn pointed_type(intrinsic: &Intrinsic) -> Result { || intrinsic.cpuid.iter().any(|cpuid| { matches!( &**cpuid, - "KEYLOCKER" | "KEYLOCKER_WIDE" | "XSAVE" | "FXSR" | "CLFLUSHOPT" + "KEYLOCKER" | "KEYLOCKER_WIDE" | "XSAVE" | "FXSR" | "CLFLUSHOPT" | "MOVDIR64B" ) }) { - // AMX, KEYLOCKER, XSAVE and CLFLUSHOPT intrinsics should take `*u8` + // AMX, KEYLOCKER, XSAVE, CLFLUSHOPT and MOVDIR64B intrinsics should take `*u8` U8 } else if intrinsic.name == "_mm_clflush" { // Just a false match in the following logic U8 + } else if intrinsic.cpuid.iter().any(|cpuid| cpuid == "MOVDIRI") { + // `_directstoreu_u32`/`_directstoreu_u64` store the integer type in their name + if intrinsic.name.ends_with("u32") { + U32 + } else { + U64 + } } else if ["_mm_storeu_si", "_mm_loadu_si"] .iter() .any(|x| intrinsic.name.starts_with(x))