From 3d963f14cf087f299c9b290798a4b9b3f9ac5307 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 20 Oct 2026 12:06:41 +0100 Subject: [PATCH 1/5] bfloat16 types --- crates/core_arch/Cargo.toml | 5 +- crates/core_arch/src/arm_shared/neon/mod.rs | 76 +++++++++++++++++++++ crates/core_arch/src/lib.rs | 4 +- crates/core_arch/src/simd.rs | 6 ++ 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/crates/core_arch/Cargo.toml b/crates/core_arch/Cargo.toml index ee3eff825c..e69705e004 100644 --- a/crates/core_arch/Cargo.toml +++ b/crates/core_arch/Cargo.toml @@ -25,7 +25,10 @@ stdarch-test = { version = "0.*", path = "../stdarch-test" } syscalls = { version = "0.6.18", default-features = false } [lints.rust] -unexpected_cfgs = { level = "warn", check-cfg = ['cfg(target_has_reliable_f16)'] } +unexpected_cfgs = { level = "warn", check-cfg = [ + 'cfg(target_has_reliable_f16b)', + 'cfg(target_has_reliable_f16)', +] } [lints.clippy] too_long_first_doc_paragraph = "allow" diff --git a/crates/core_arch/src/arm_shared/neon/mod.rs b/crates/core_arch/src/arm_shared/neon/mod.rs index 3af59b1dfb..6d204c55df 100644 --- a/crates/core_arch/src/arm_shared/neon/mod.rs +++ b/crates/core_arch/src/arm_shared/neon/mod.rs @@ -48,6 +48,12 @@ pub(crate) type p16 = u16; pub(crate) type p64 = u64; pub(crate) type p128 = u128; +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[allow(non_camel_case_types)] +#[unstable(feature = "f16b", issue = "160630")] +pub(crate) type bfloat16_t = core::num::f16b; + types! { #![cfg_attr(not(target_arch = "arm"), stable(feature = "neon_intrinsics", since = "1.59.0"))] #![cfg_attr(target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800"))] @@ -113,6 +119,18 @@ types! { pub struct float16x8_t(8 x pub(crate) f16); } +#[cfg(any(target_has_reliable_f16b, doc))] +types! { + /// Arm-specific 64-bit wide vector of four packed `f16b`. + #[doc(cfg(target_has_reliable_f16b))] + stability: [unstable(feature = "f16b", issue = "160630")] + pub struct bfloat16x4_t(4 x pub(crate) bfloat16_t); + /// Arm-specific 128-bit wide vector of eight packed `f16b`. + #[doc(cfg(target_has_reliable_f16b))] + stability: [unstable(feature = "f16b", issue = "160630")] + pub struct bfloat16x8_t(8 x pub(crate) bfloat16_t); +} + /// Arm-specific type containing two `int8x8_t` vectors. #[repr(C)] #[derive(Copy, Clone, Debug)] @@ -833,6 +851,64 @@ pub struct float16x8x4_t( pub float16x8_t, ); +/// Arm-specific type containing two `bfloat16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "f16b", issue = "160630")] +pub struct bfloat16x4x2_t(pub bfloat16x4_t, pub bfloat16x4_t); + +/// Arm-specific type containing three `bfloat16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "f16b", issue = "160630")] +pub struct bfloat16x4x3_t(pub bfloat16x4_t, pub bfloat16x4_t, pub bfloat16x4_t); + +/// Arm-specific type containing four `bfloat16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "f16b", issue = "160630")] +pub struct bfloat16x4x4_t( + pub bfloat16x4_t, + pub bfloat16x4_t, + pub bfloat16x4_t, + pub bfloat16x4_t, +); + +/// Arm-specific type containing two `bfloat16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "f16b", issue = "160630")] +pub struct bfloat16x8x2_t(pub bfloat16x8_t, pub bfloat16x8_t); + +/// Arm-specific type containing three `bfloat16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "f16b", issue = "160630")] +pub struct bfloat16x8x3_t(pub bfloat16x8_t, pub bfloat16x8_t, pub bfloat16x8_t); + +/// Arm-specific type containing four `bfloat16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "f16b", issue = "160630")] +pub struct bfloat16x8x4_t( + pub bfloat16x8_t, + pub bfloat16x8_t, + pub bfloat16x8_t, + pub bfloat16x8_t, +); + /// Arm-specific type containing two `float32x2_t` vectors. #[repr(C)] #[derive(Copy, Clone, Debug)] diff --git a/crates/core_arch/src/lib.rs b/crates/core_arch/src/lib.rs index c7e2d93b88..2af8a830cd 100644 --- a/crates/core_arch/src/lib.rs +++ b/crates/core_arch/src/lib.rs @@ -40,7 +40,9 @@ movrs_target_feature, clflushopt_target_feature, min_adt_const_params, - num_internals + num_internals, + f16b, + cfg_target_has_reliable_f16b )] #![cfg_attr( any(target_arch = "powerpc", target_arch = "powerpc64"), diff --git a/crates/core_arch/src/simd.rs b/crates/core_arch/src/simd.rs index 08ee17548a..286f9b3bbc 100644 --- a/crates/core_arch/src/simd.rs +++ b/crates/core_arch/src/simd.rs @@ -37,6 +37,8 @@ unsafe impl SimdElement for i64 {} unsafe impl SimdElement for i128 {} unsafe impl SimdElement for f16 {} +#[cfg(any(target_has_reliable_f16b, doc))] +unsafe impl SimdElement for core::num::f16b {} unsafe impl SimdElement for f32 {} unsafe impl SimdElement for f64 {} @@ -342,6 +344,8 @@ pub(crate) type i16x4 = Simd; pub(crate) type i32x2 = Simd; pub(crate) type i64x1 = Simd; +#[cfg(any(target_has_reliable_f16b, doc))] +pub(crate) type f16bx4 = Simd; pub(crate) type f16x4 = Simd; pub(crate) type f32x2 = Simd; pub(crate) type f64x1 = Simd; @@ -358,6 +362,8 @@ pub(crate) type i16x8 = Simd; pub(crate) type i32x4 = Simd; pub(crate) type i64x2 = Simd; +#[cfg(any(target_has_reliable_f16b, doc))] +pub(crate) type f16bx8 = Simd; pub(crate) type f16x8 = Simd; pub(crate) type f32x4 = Simd; pub(crate) type f64x2 = Simd; From 051ae58e2801531e4ae4ed342e1d29e958e23f46 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 20 Oct 2026 12:06:41 +0100 Subject: [PATCH 2/5] Update Arm intrinsic generator --- crates/stdarch-gen-arm/src/fn_suffix.rs | 51 ++++++++++++++++--------- crates/stdarch-gen-arm/src/typekinds.rs | 32 ++++++++++++++-- 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/crates/stdarch-gen-arm/src/fn_suffix.rs b/crates/stdarch-gen-arm/src/fn_suffix.rs index 6fba3dc744..3b27b533b0 100644 --- a/crates/stdarch-gen-arm/src/fn_suffix.rs +++ b/crates/stdarch-gen-arm/src/fn_suffix.rs @@ -44,7 +44,7 @@ pub fn type_to_size(str_type: &str) -> i32 { "int8x8_t" | "int8x16_t" | "i8" | "s8" | "uint8x8_t" | "uint8x16_t" | "u8" | "poly8x8_t" | "poly8x16_t" => 8, "int16x4_t" | "int16x8_t" | "i16" | "s16" | "uint16x4_t" | "uint16x8_t" | "u16" - | "float16x4_t" | "float16x8_t" | "_f16" | "poly16x4_t" | "poly16x8_t" => 16, + | "_bf16" | "float16x4_t" | "float16x8_t" | "_f16" | "poly16x4_t" | "poly16x8_t" => 16, "int32x2_t" | "int32x4_t" | "i32" | "s32" | "uint32x2_t" | "uint32x4_t" | "u32" | "float32x2_t" | "float32x4_t" | "f32" => 32, "int64x1_t" | "int64x2_t" | "i64" | "s64" | "uint64x1_t" | "uint64x2_t" | "u64" @@ -54,9 +54,17 @@ pub fn type_to_size(str_type: &str) -> i32 { } } +fn is_bfloat(ty: &VectorType) -> bool { + match ty.base_type() { + BaseType::Sized(BaseTypeKind::BFloat, _) => true, + _ => false, + } +} + fn neon_get_base_and_char(ty: &VectorType) -> (u32, char, bool) { let lanes = ty.lanes(); match ty.base_type() { + BaseType::Sized(BaseTypeKind::BFloat, size) => (*size, 'b', *size * lanes == 128), BaseType::Sized(BaseTypeKind::Float, size) => (*size, 'f', *size * lanes == 128), BaseType::Sized(BaseTypeKind::Int, size) => (*size, 's', *size * lanes == 128), BaseType::Sized(BaseTypeKind::UInt, size) => (*size, 'u', *size * lanes == 128), @@ -73,11 +81,16 @@ pub fn make_neon_suffix(type_kind: TypeKind, suffix_kind: SuffixKind) -> String TypeKind::Vector(ty) => { let tuple_size = ty.tuple_size().map_or(0, |t| t.to_int()); let (base_size, prefix_char, requires_q) = neon_get_base_and_char(&ty); + let prefix_str = if is_bfloat(&ty) { + String::from("bf") + } else { + String::from(prefix_char) + }; let prefix_q = if requires_q { "q" } else { "" }; let lanes = ty.lanes(); match suffix_kind { SuffixKind::Normal => { - let mut str_suffix: String = format!("{prefix_q}_{prefix_char}{base_size}"); + let mut str_suffix: String = format!("{prefix_q}_{prefix_str}{base_size}"); if tuple_size > 0 { str_suffix.push_str("_x"); str_suffix.push_str(tuple_size.to_string().as_str()); @@ -85,11 +98,11 @@ pub fn make_neon_suffix(type_kind: TypeKind, suffix_kind: SuffixKind) -> String str_suffix } SuffixKind::NSuffix => { - format!("{prefix_q}_n_{prefix_char}{base_size}") + format!("{prefix_q}_n_{prefix_str}{base_size}") } - SuffixKind::NoQ => format!("_{prefix_char}{base_size}"), - SuffixKind::NoQNSuffix => format!("_n{prefix_char}{base_size}"), + SuffixKind::NoQ => format!("_{prefix_str}{base_size}"), + SuffixKind::NoQNSuffix => format!("_n{prefix_str}{base_size}"), SuffixKind::Unsigned => { let t = type_kind.to_string(); @@ -114,10 +127,10 @@ pub fn make_neon_suffix(type_kind: TypeKind, suffix_kind: SuffixKind) -> String } SuffixKind::Base => base_size.to_string(), SuffixKind::NoX => { - format!("{prefix_q}_{prefix_char}{base_size}") + format!("{prefix_q}_{prefix_str}{base_size}") } SuffixKind::Dup => { - let mut str_suffix: String = format!("{prefix_q}_dup_{prefix_char}{base_size}"); + let mut str_suffix: String = format!("{prefix_q}_dup_{prefix_str}{base_size}"); if tuple_size > 0 { str_suffix.push_str("_x"); str_suffix.push_str(tuple_size.to_string().as_str()); @@ -125,40 +138,40 @@ pub fn make_neon_suffix(type_kind: TypeKind, suffix_kind: SuffixKind) -> String str_suffix } SuffixKind::DupNox => { - format!("{prefix_q}_dup_{prefix_char}{base_size}") + format!("{prefix_q}_dup_{prefix_str}{base_size}") } SuffixKind::LaneNoX => { - format!("{prefix_q}_lane_{prefix_char}{base_size}") + format!("{prefix_q}_lane_{prefix_str}{base_size}") } SuffixKind::LaneQNoX => { - format!("{prefix_q}_laneq_{prefix_char}{base_size}") + format!("{prefix_q}_laneq_{prefix_str}{base_size}") } SuffixKind::Rot270 => { - format!("{prefix_q}_rot270_{prefix_char}{base_size}") + format!("{prefix_q}_rot270_{prefix_str}{base_size}") } SuffixKind::Rot270Lane => { - format!("{prefix_q}_rot270_lane_{prefix_char}{base_size}") + format!("{prefix_q}_rot270_lane_{prefix_str}{base_size}") } SuffixKind::Rot270LaneQ => { - format!("{prefix_q}_rot270_laneq_{prefix_char}{base_size}") + format!("{prefix_q}_rot270_laneq_{prefix_str}{base_size}") } SuffixKind::Rot180 => { - format!("{prefix_q}_rot180_{prefix_char}{base_size}") + format!("{prefix_q}_rot180_{prefix_str}{base_size}") } SuffixKind::Rot180Lane => { - format!("{prefix_q}_rot180_lane_{prefix_char}{base_size}") + format!("{prefix_q}_rot180_lane_{prefix_str}{base_size}") } SuffixKind::Rot180LaneQ => { - format!("{prefix_q}_rot180_laneq_{prefix_char}{base_size}") + format!("{prefix_q}_rot180_laneq_{prefix_str}{base_size}") } SuffixKind::Rot90 => { - format!("{prefix_q}_rot90_{prefix_char}{base_size}") + format!("{prefix_q}_rot90_{prefix_str}{base_size}") } SuffixKind::Rot90Lane => { - format!("{prefix_q}_rot90_lane_{prefix_char}{base_size}") + format!("{prefix_q}_rot90_lane_{prefix_str}{base_size}") } SuffixKind::Rot90LaneQ => { - format!("{prefix_q}_rot90_laneq_{prefix_char}{base_size}") + format!("{prefix_q}_rot90_laneq_{prefix_str}{base_size}") } SuffixKind::BaseByteSize => format!("{}", base_size / 8), } diff --git a/crates/stdarch-gen-arm/src/typekinds.rs b/crates/stdarch-gen-arm/src/typekinds.rs index c3aa22294d..f833cb3d5d 100644 --- a/crates/stdarch-gen-arm/src/typekinds.rs +++ b/crates/stdarch-gen-arm/src/typekinds.rs @@ -77,7 +77,7 @@ impl TypeKindOptions { BaseTypeKind::Int => self.s, BaseTypeKind::UInt => self.u, BaseTypeKind::Poly => self.p, - BaseTypeKind::Bool => false, + BaseTypeKind::BFloat | BaseTypeKind::Bool => false, } } } @@ -113,6 +113,7 @@ impl fmt::Display for TypeKindOptions { #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum BaseTypeKind { Float, + BFloat, Int, UInt, Bool, @@ -497,7 +498,7 @@ impl FromStr for VectorType { fn from_str(s: &str) -> Result { static RE: LazyLock = LazyLock::new(|| { - Regex::new(r"^(?:(?:sv(?P(?:uint|int|bool|float)(?:\d+)?))|(?:(?P(?:uint|int|bool|poly|float)(?:\d+)?)x(?P(?:\d+)?)))(?:x(?P2|3|4))?_t$").unwrap() + Regex::new(r"^(?:(?:sv(?P(?:uint|int|bool|float|bfloat)(?:\d+)?))|(?:(?P(?:uint|int|bool|poly|float|bfloat)(?:\d+)?)x(?P(?:\d+)?)))(?:x(?P2|3|4))?_t$").unwrap() }); if let Some(c) = RE.captures(s) { @@ -634,6 +635,7 @@ impl FromStr for BaseTypeKind { fn from_str(s: &str) -> Result { match s { "float" | "f" => Ok(Self::Float), + "bfloat" | "bf16" => Ok(Self::BFloat), "int" | "i" => Ok(Self::Int), "uint" | "u" => Ok(Self::UInt), "poly" | "p" => Ok(Self::Poly), @@ -647,9 +649,14 @@ impl ToRepr for BaseTypeKind { fn repr(&self, repr: TypeRepr) -> String { match (repr, self) { (TypeRepr::C, Self::Float) => "float", + (TypeRepr::C, Self::BFloat) => "bfloat16", (TypeRepr::C, Self::Int) => "int", (TypeRepr::C, Self::UInt) => "uint", (TypeRepr::C, Self::Poly) => "poly", + + (TypeRepr::Rust, Self::BFloat) => "bfloat16_t", + (TypeRepr::LLVMMachine | TypeRepr::ACLENotation, Self::BFloat) => "bf16", + (TypeRepr::Rust | TypeRepr::LLVMMachine | TypeRepr::ACLENotation, Self::Float) => "f", (TypeRepr::Rust, Self::Int) | (TypeRepr::LLVMMachine, Self::Int | Self::UInt) => "i", (TypeRepr::Rust | TypeRepr::ACLENotation, Self::UInt) => "u", @@ -692,6 +699,10 @@ impl BaseType { pub fn is_float(&self) -> bool { self.kind() == &BaseTypeKind::Float } + + pub fn is_bfloat(&self) -> bool { + self.kind() == &BaseTypeKind::BFloat + } } impl FromStr for BaseType { @@ -709,6 +720,13 @@ impl FromStr for BaseType { .map(u32::from_str) .transpose() .unwrap(); + + if kind == BaseTypeKind::BFloat && size != Some(16) { + return Err(format!( + "a `bfloat` must have a size of `16`, got `{size:?}`" + )); + } + match size { Some(size) => Ok(Self::Sized(kind, size)), None => Ok(Self::Unsized(kind)), @@ -733,6 +751,7 @@ impl ToRepr for BaseType { (Sized(_, size), SizeLiteral) if *size == 128 => "q".to_string(), (_, SizeLiteral) => unreachable!("cannot represent {self:#?} as size literal"), (Sized(Float, _) | Unsized(Float), TypeKind) => "f".to_string(), + (Sized(BFloat, _) | Unsized(BFloat), TypeKind) => "bf16".to_string(), (Sized(Int, _) | Unsized(Int), TypeKind) => "s".to_string(), (Sized(UInt, _) | Unsized(UInt), TypeKind) => "u".to_string(), (Sized(_, size), Size) => size.to_string(), @@ -740,7 +759,14 @@ impl ToRepr for BaseType { assert!(size.is_power_of_two() && *size >= 8); (size >> 3).trailing_zeros().to_string() } - (Sized(kind, size), _) => format!("{}{size}", kind.repr(repr)), + (Sized(kind, size), _) => { + if self.is_bfloat() { + assert!(*size == 16); + format!("{}", kind.repr(repr)) + } else { + format!("{}{size}", kind.repr(repr)) + } + } (Unsized(kind), _) => kind.repr(repr), } } From dda268c2339e9da6ac0e4da85ad587ddaaf97183 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 20 Oct 2026 12:06:41 +0100 Subject: [PATCH 3/5] Update test harness --- .../missing_aarch64_be_common.txt | 6 + .../intrinsic-test/missing_aarch64_common.txt | 6 + crates/intrinsic-test/src/arm/mod.rs | 13 +- crates/intrinsic-test/src/arm/types.rs | 6 + crates/intrinsic-test/src/common/gen_rust.rs | 22 +++ .../src/common/intrinsic_helpers.rs | 3 + crates/intrinsic-test/src/common/values.rs | 5 +- crates/stdarch-verify/src/lib.rs | 9 + crates/stdarch-verify/tests/arm.rs | 181 ++++++++++++++++++ 9 files changed, 244 insertions(+), 7 deletions(-) diff --git a/crates/intrinsic-test/missing_aarch64_be_common.txt b/crates/intrinsic-test/missing_aarch64_be_common.txt index 327c8207a0..9f22e528ba 100644 --- a/crates/intrinsic-test/missing_aarch64_be_common.txt +++ b/crates/intrinsic-test/missing_aarch64_be_common.txt @@ -14,12 +14,14 @@ vscale_f32 vscaleq_f16 vscaleq_f32 vscaleq_f64 +vluti2_lane_bf16 vluti2_lane_p16 vluti2_lane_p8 vluti2_lane_s16 vluti2_lane_s8 vluti2_lane_u16 vluti2_lane_u8 +vluti2q_lane_bf16 vluti2q_lane_p16 vluti2q_lane_p8 vluti2q_lane_s16 @@ -28,6 +30,7 @@ vluti2q_lane_u16 vluti2_laneq_f16 vluti2_lane_f16 vluti2_laneq_f16 +vluti2_laneq_bf16 vluti2_laneq_p16 vluti2_laneq_p8 vluti2_laneq_s16 @@ -36,6 +39,7 @@ vluti2_laneq_u16 vluti2_laneq_u8 vluti2q_lane_f16 vluti2q_laneq_f16 +vluti2q_laneq_bf16 vluti2q_laneq_p16 vluti2q_laneq_p8 vluti2q_laneq_s16 @@ -44,6 +48,7 @@ vluti2q_laneq_u16 vluti2q_laneq_u8 vluti2q_lane_u8 vluti4q_lane_f16_x2 +vluti4q_lane_bf16_x2 vluti4q_lane_p16_x2 vluti4q_lane_p8 vluti4q_lane_s16_x2 @@ -51,6 +56,7 @@ vluti4q_lane_s8 vluti4q_lane_u16_x2 vluti4q_lane_u8 vluti4q_laneq_f16_x2 +vluti4q_laneq_bf16_x2 vluti4q_laneq_p16_x2 vluti4q_laneq_p8 vluti4q_laneq_s16_x2 diff --git a/crates/intrinsic-test/missing_aarch64_common.txt b/crates/intrinsic-test/missing_aarch64_common.txt index 327c8207a0..f154f90442 100644 --- a/crates/intrinsic-test/missing_aarch64_common.txt +++ b/crates/intrinsic-test/missing_aarch64_common.txt @@ -14,17 +14,20 @@ vscale_f32 vscaleq_f16 vscaleq_f32 vscaleq_f64 +vluti2_lane_bf16 vluti2_lane_p16 vluti2_lane_p8 vluti2_lane_s16 vluti2_lane_s8 vluti2_lane_u16 vluti2_lane_u8 +vluti2q_lane_bf16 vluti2q_lane_p16 vluti2q_lane_p8 vluti2q_lane_s16 vluti2q_lane_s8 vluti2q_lane_u16 +vluti2_laneq_bf16 vluti2_laneq_f16 vluti2_lane_f16 vluti2_laneq_f16 @@ -35,6 +38,7 @@ vluti2_laneq_s8 vluti2_laneq_u16 vluti2_laneq_u8 vluti2q_lane_f16 +vluti2q_laneq_bf16 vluti2q_laneq_f16 vluti2q_laneq_p16 vluti2q_laneq_p8 @@ -43,6 +47,7 @@ vluti2q_laneq_s8 vluti2q_laneq_u16 vluti2q_laneq_u8 vluti2q_lane_u8 +vluti4q_lane_bf16_x2 vluti4q_lane_f16_x2 vluti4q_lane_p16_x2 vluti4q_lane_p8 @@ -50,6 +55,7 @@ vluti4q_lane_s16_x2 vluti4q_lane_s8 vluti4q_lane_u16_x2 vluti4q_lane_u8 +vluti4q_laneq_bf16_x2 vluti4q_laneq_f16_x2 vluti4q_laneq_p16_x2 vluti4q_laneq_p8 diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index 5e892c6134..38f57310a3 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -45,8 +45,8 @@ impl SupportedArchitecture for Arm { let a32 = cli_options.target.starts_with("armv7"); match cli_options.cc_arg_style { CcArgStyle::Clang if !a32 && !big_endian => vec![ - "-march=armv8.6a+crypto+crc+dotprod+fp16+sve2-aes+sve2-sm4+sve2-sha3+sve2-bitperm+\ - f32mm+f64mm+sve2p1", + "-march=armv8.6a+crypto+crc+dotprod+fp16+bf16+sve2-aes+sve2-sm4+sve2-sha3+\ + sve2-bitperm+f32mm+f64mm+sve2p1", ], CcArgStyle::Clang => vec!["-march=armv8.6a+crypto+crc+dotprod+fp16"], // SVE tests aren't run under GCC so there are no target features added for SVE @@ -70,9 +70,6 @@ impl SupportedArchitecture for Arm { .into_iter() // Skip intrinsics that don't return a value. .filter(|i| i.results.kind() != TypeKind::Void) - // Skip bfloat intrinsics - not currently supported - .filter(|i| i.results.kind() != TypeKind::BFloat) - .filter(|i| !i.arguments.iter().any(|a| a.ty.kind() == TypeKind::BFloat)) // Skip SVE intrinsics that have `f16` - not yet implemented! .filter(|i| { let has_f16_arg = i @@ -121,6 +118,8 @@ impl SupportedArchitecture for Arm { .filter(|i| !i.arguments.iter().any(|a| a.is_ptr())) // Skip intrinsics with 128-bit elements (e.g. `p128`) .filter(|i| !i.arguments.iter().any(|a| a.ty.inner_size() == 128)) + // A32 does not provide polynomial 128-bit return types either. + .filter(|i| !(a32 && i.results.inner_size() == 128)) // Skip intrinsics from `--skip` .filter(|i| !cli_options.skip.contains(&i.name)) // Skip A64-specific intrinsics on A32 @@ -178,9 +177,13 @@ const RUST_PRELUDE: &str = r#" #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_ftts))] #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_feat_lut))] #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_fp8))] +#![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_bf16))] #![cfg_attr(all(any(target_arch = "aarch64", target_arch = "arm64ec"), target_endian = "little"), feature(stdarch_aarch64_sve))] #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(faminmax))] #![feature(stdarch_neon_f16)] +#![feature(f16b)] + +use core::num::f16b; #[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))] use core_arch::arch::aarch64::*; diff --git a/crates/intrinsic-test/src/arm/types.rs b/crates/intrinsic-test/src/arm/types.rs index cced3bc40c..b14a6b7d83 100644 --- a/crates/intrinsic-test/src/arm/types.rs +++ b/crates/intrinsic-test/src/arm/types.rs @@ -39,6 +39,12 @@ impl TypeDefinition for ArmType { } fn rust_type(&self) -> String { + // Scalar BF16 uses Rust's `f16b` spelling rather than `bf16`. This also + // gives the comparison wrapper its expected `NanEqF16b` name. + if self.kind() == TypeKind::BFloat && self.simd_len.is_none() && self.vec_len.is_none() { + return "f16b".to_string(); + } + let rust_prefix = self.kind.rust_prefix(); let c_prefix = self.kind.c_prefix(); diff --git a/crates/intrinsic-test/src/common/gen_rust.rs b/crates/intrinsic-test/src/common/gen_rust.rs index f319418b49..350744e53f 100644 --- a/crates/intrinsic-test/src/common/gen_rust.rs +++ b/crates/intrinsic-test/src/common/gen_rust.rs @@ -31,6 +31,23 @@ macro_rules! wrap_partialeq { } wrap_partialeq!(NanEqF16(f16), NanEqF32(f32), NanEqF64(f64)); + +#[cfg(target_has_reliable_f16b)] +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub struct NanEqF16b(f16b); + +#[cfg(target_has_reliable_f16b)] +impl PartialEq for NanEqF16b { + fn eq(&self, other: &Self) -> bool { + let a = f32::from(self.0); + let b = f32::from(other.0); + a == b || (a.is_nan() && b.is_nan()) + } +} + +#[cfg(target_has_reliable_f16b)] +impl Eq for NanEqF16b {} "#; /// Run rustfmt on the generated source code @@ -131,6 +148,9 @@ core_arch = {{ path = "../../crates/core_arch" }} [build-dependencies] cc = "1" + +[lints.rust] +unexpected_cfgs = {{ level = "warn", check-cfg = ['cfg(target_has_reliable_f16b)'] }} "#, authors = env!("CARGO_PKG_AUTHORS") .split(":") @@ -165,6 +185,8 @@ pub fn write_lib_rs( {notice} #![feature(simd_ffi)] #![feature(f16)] +#![feature(cfg_target_has_reliable_f16b)] +#![allow(internal_features)] #![allow(unused)] // Cargo will spam the logs if these warnings are not silenced. diff --git a/crates/intrinsic-test/src/common/intrinsic_helpers.rs b/crates/intrinsic-test/src/common/intrinsic_helpers.rs index bc47a90d3f..76da7ca80b 100644 --- a/crates/intrinsic-test/src/common/intrinsic_helpers.rs +++ b/crates/intrinsic-test/src/common/intrinsic_helpers.rs @@ -79,6 +79,7 @@ impl TypeKind { pub fn c_prefix(&self) -> &str { match self { Self::Bool => "bool", + Self::BFloat => "bfloat", Self::Float => "float", Self::Int(Sign::Signed) => "int", Self::Int(Sign::Unsigned) => "uint", @@ -253,6 +254,8 @@ pub(crate) fn default_fixed_vector_comparison( ), ")", ) + } else if ty.kind == TypeKind::BFloat { + ("NanEqF16b(".to_string(), ")") } else if ty.kind == TypeKind::Float { ( match ty.inner_size() { diff --git a/crates/intrinsic-test/src/common/values.rs b/crates/intrinsic-test/src/common/values.rs index d1f333fd19..88d217b288 100644 --- a/crates/intrinsic-test/src/common/values.rs +++ b/crates/intrinsic-test/src/common/values.rs @@ -61,10 +61,10 @@ pub fn test_values_array(ty: &IntrinsicType) -> String { .. } => (1, TypeKind::Bool), IntrinsicType { - kind: TypeKind::Float, + kind: kind @ (TypeKind::Float | TypeKind::BFloat), bit_len: Some(bit_len), .. - } => (*bit_len, TypeKind::Float), + } => (*bit_len, *kind), IntrinsicType { kind: TypeKind::Vector, .. @@ -87,6 +87,7 @@ pub fn test_values_array(ty: &IntrinsicType) -> String { fmt(&format_args!("{}", if src == 1 { true } else { false })) } TypeKind::Float => fmt(&format_args!("f{bit_len}::from_bits({src:#x})")), + TypeKind::BFloat => fmt(&format_args!("f16b::from_bits({src:#x})")), TypeKind::Vector | TypeKind::Int(Sign::Signed) if (src >> (bit_len - 1)) != 0 => { // `src` is a two's complement representation of a negative value. let mask = !0u64 >> (64 - bit_len); diff --git a/crates/stdarch-verify/src/lib.rs b/crates/stdarch-verify/src/lib.rs index 5412ab466a..98e174cd47 100644 --- a/crates/stdarch-verify/src/lib.rs +++ b/crates/stdarch-verify/src/lib.rs @@ -205,6 +205,7 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { "__tile1024i" => quote! { &TILE1024I }, "bool" => quote! { &BOOL }, "bf16" => quote! { &BF16 }, + "f16b" | "bfloat16_t" => quote! { &BF16 }, "f16" => quote! { &F16 }, "f32" => quote! { &F32 }, "f64" => quote! { &F64 }, @@ -292,6 +293,14 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { "uint64x2x3_t" => quote! { &U64X2X3 }, "uint64x2x4_t" => quote! { &U64X2X4 }, "float16x2_t" => quote! { &F16X2 }, + "bfloat16x4_t" => quote! { &BF16X4 }, + "bfloat16x4x2_t" => quote! { &BF16X4X2 }, + "bfloat16x4x3_t" => quote! { &BF16X4X3 }, + "bfloat16x4x4_t" => quote! { &BF16X4X4 }, + "bfloat16x8_t" => quote! { &BF16X8 }, + "bfloat16x8x2_t" => quote! { &BF16X8X2 }, + "bfloat16x8x3_t" => quote! { &BF16X8X3 }, + "bfloat16x8x4_t" => quote! { &BF16X8X4 }, "float16x4_t" => quote! { &F16X4 }, "float16x4x2_t" => quote! { &F16X4X2 }, "float16x4x3_t" => quote! { &F16X4X3 }, diff --git a/crates/stdarch-verify/tests/arm.rs b/crates/stdarch-verify/tests/arm.rs index 6ba9c17c48..d2934bdc40 100644 --- a/crates/stdarch-verify/tests/arm.rs +++ b/crates/stdarch-verify/tests/arm.rs @@ -17,6 +17,7 @@ struct Function { } static BOOL: Type = Type::PrimBool; +static BF16: Type = Type::BFloat16; static F16: Type = Type::PrimFloat(16); static F32: Type = Type::PrimFloat(32); static F64: Type = Type::PrimFloat(64); @@ -34,6 +35,14 @@ static GENERICT: Type = Type::GenericParam("T"); static GENERICU: Type = Type::GenericParam("U"); static F16X4: Type = Type::F(16, 4, 1); +static BF16X4: Type = Type::BF(4, 1); +static BF16X4X2: Type = Type::BF(4, 2); +static BF16X4X3: Type = Type::BF(4, 3); +static BF16X4X4: Type = Type::BF(4, 4); +static BF16X8: Type = Type::BF(8, 1); +static BF16X8X2: Type = Type::BF(8, 2); +static BF16X8X3: Type = Type::BF(8, 3); +static BF16X8X4: Type = Type::BF(8, 4); static F16X4X2: Type = Type::F(16, 4, 2); static F16X4X3: Type = Type::F(16, 4, 3); static F16X4X4: Type = Type::F(16, 4, 4); @@ -209,6 +218,7 @@ static SVPATTERN: Type = Type::Enum("svpattern"); enum Type { Void, PrimBool, + BFloat16, PrimFloat(u8), PrimSigned(u8), PrimUnsigned(u8), @@ -221,6 +231,7 @@ enum Type { U(u8, u8, u8), P(u8, u8, u8), F(u8, u8, u8), + BF(u8, u8), Pred(u8), SVI(u8, u8), SVU(u8, u8), @@ -276,6 +287,10 @@ fn verify_all_signatures() { "vreinterpretq_p64_p128", "vreinterpretq_p128_p64", "vreinterpretq_f32_p128", + "vreinterpret_bf16_f16", + "vreinterpret_f16_bf16", + "vreinterpretq_bf16_f16", + "vreinterpretq_f16_bf16", "vtst_p16", "vtstq_p16", "__dbg", @@ -489,6 +504,15 @@ fn parse_ty_base(s: &str) -> &'static Type { match s { "bool" => &BOOL, "void" => &VOID, + "bfloat16_t" => &BF16, + "bfloat16x4_t" => &BF16X4, + "bfloat16x4x2_t" => &BF16X4X2, + "bfloat16x4x3_t" => &BF16X4X3, + "bfloat16x4x4_t" => &BF16X4X4, + "bfloat16x8_t" => &BF16X8, + "bfloat16x8x2_t" => &BF16X8X2, + "bfloat16x8x3_t" => &BF16X8X3, + "bfloat16x8x4_t" => &BF16X8X4, "float16_t" => &F16, "float16x4_t" => &F16X4, "float16x4x2_t" => &F16X4X2, @@ -683,6 +707,163 @@ static SKIP_ASSERT_INSTR_TESTS: &'static [&'static str] = &["svpfalse_b"]; // are no longer being added and just adding to this list indefinitely isn't the best solution for // dealing with that. static SKIP_RUNTIME_TESTS: &'static [&'static str] = &[ + "vbfdot_f32", + "vbfdot_lane_f32", + "vbfdot_laneq_f32", + "vbfdotq_f32", + "vbfdotq_lane_f32", + "vbfdotq_laneq_f32", + "vbfmlalbq_f32", + "vbfmlalbq_lane_f32", + "vbfmlalbq_laneq_f32", + "vbfmlaltq_f32", + "vbfmlaltq_lane_f32", + "vbfmlaltq_laneq_f32", + "vbfmmlaq_f32", + "vcombine_bf16", + "vcopy_lane_bf16", + "vcopy_laneq_bf16", + "vcopyq_lane_bf16", + "vcopyq_laneq_bf16", + "vcreate_bf16", + "vcvt_bf16_f32", + "vcvt_f32_bf16", + "vcvtah_f32_bf16", + "vcvth_bf16_f32", + "vcvtq_high_bf16_f32", + "vcvtq_high_f32_bf16", + "vcvtq_low_bf16_f32", + "vcvtq_low_f32_bf16", + "vdup_lane_bf16", + "vdup_laneq_bf16", + "vdup_n_bf16", + "vduph_lane_bf16", + "vduph_laneq_bf16", + "vdupq_lane_bf16", + "vdupq_laneq_bf16", + "vdupq_n_bf16", + "vget_high_bf16", + "vget_lane_bf16", + "vget_low_bf16", + "vgetq_lane_bf16", + "vld1_bf16", + "vld1_bf16_x2", + "vld1_bf16_x3", + "vld1_bf16_x4", + "vld1_dup_bf16", + "vld1_lane_bf16", + "vld1q_bf16", + "vld1q_bf16_x2", + "vld1q_bf16_x3", + "vld1q_bf16_x4", + "vld1q_dup_bf16", + "vld1q_lane_bf16", + "vld2_bf16", + "vld2_dup_bf16", + "vld2_lane_bf16", + "vld2q_bf16", + "vld2q_dup_bf16", + "vld2q_lane_bf16", + "vld3_bf16", + "vld3_dup_bf16", + "vld3_lane_bf16", + "vld3q_bf16", + "vld3q_dup_bf16", + "vld3q_lane_bf16", + "vld4_bf16", + "vld4_dup_bf16", + "vld4_lane_bf16", + "vld4q_bf16", + "vld4q_dup_bf16", + "vld4q_lane_bf16", + "vluti2_lane_bf16", + "vluti2_laneq_bf16", + "vluti2q_lane_bf16", + "vluti2q_laneq_bf16", + "vluti4q_lane_bf16_x2", + "vluti4q_laneq_bf16_x2", + "vreinterpret_bf16_f16", + "vreinterpret_bf16_f32", + "vreinterpret_bf16_f64", + "vreinterpret_bf16_p16", + "vreinterpret_bf16_p64", + "vreinterpret_bf16_p8", + "vreinterpret_bf16_s16", + "vreinterpret_bf16_s32", + "vreinterpret_bf16_s64", + "vreinterpret_bf16_s8", + "vreinterpret_bf16_u16", + "vreinterpret_bf16_u32", + "vreinterpret_bf16_u64", + "vreinterpret_bf16_u8", + "vreinterpret_f16_bf16", + "vreinterpret_f32_bf16", + "vreinterpret_f64_bf16", + "vreinterpret_p16_bf16", + "vreinterpret_p64_bf16", + "vreinterpret_p8_bf16", + "vreinterpret_s16_bf16", + "vreinterpret_s32_bf16", + "vreinterpret_s64_bf16", + "vreinterpret_s8_bf16", + "vreinterpret_u16_bf16", + "vreinterpret_u32_bf16", + "vreinterpret_u64_bf16", + "vreinterpret_u8_bf16", + "vreinterpretq_bf16_f16", + "vreinterpretq_bf16_f32", + "vreinterpretq_bf16_f64", + "vreinterpretq_bf16_p128", + "vreinterpretq_bf16_p16", + "vreinterpretq_bf16_p64", + "vreinterpretq_bf16_p8", + "vreinterpretq_bf16_s16", + "vreinterpretq_bf16_s32", + "vreinterpretq_bf16_s64", + "vreinterpretq_bf16_s8", + "vreinterpretq_bf16_u16", + "vreinterpretq_bf16_u32", + "vreinterpretq_bf16_u64", + "vreinterpretq_bf16_u8", + "vreinterpretq_f16_bf16", + "vreinterpretq_f32_bf16", + "vreinterpretq_f64_bf16", + "vreinterpretq_p128_bf16", + "vreinterpretq_p16_bf16", + "vreinterpretq_p64_bf16", + "vreinterpretq_p8_bf16", + "vreinterpretq_s16_bf16", + "vreinterpretq_s32_bf16", + "vreinterpretq_s64_bf16", + "vreinterpretq_s8_bf16", + "vreinterpretq_u16_bf16", + "vreinterpretq_u32_bf16", + "vreinterpretq_u64_bf16", + "vreinterpretq_u8_bf16", + "vset_lane_bf16", + "vsetq_lane_bf16", + "vst1_bf16", + "vst1_bf16_x2", + "vst1_bf16_x3", + "vst1_bf16_x4", + "vst1_lane_bf16", + "vst1q_bf16", + "vst1q_bf16_x2", + "vst1q_bf16_x3", + "vst1q_bf16_x4", + "vst1q_lane_bf16", + "vst2_bf16", + "vst2_lane_bf16", + "vst2q_bf16", + "vst2q_lane_bf16", + "vst3_bf16", + "vst3_lane_bf16", + "vst3q_bf16", + "vst3q_lane_bf16", + "vst4_bf16", + "vst4_lane_bf16", + "vst4q_bf16", + "vst4q_lane_bf16", "vaddq_s64", "vaddq_u64", "vrsqrte_f32", From 92af949bb76950877bcde30face27496e11ef090 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 20 Oct 2026 12:06:41 +0100 Subject: [PATCH 4/5] Add bf16 intrinsics to YAML, JSON & re-generate --- .../core_arch/src/aarch64/neon/generated.rs | 256 + .../src/arm_shared/neon/generated.rs | 4944 ++++++++++- .../spec/neon/aarch64.spec.yml | 193 + .../spec/neon/arm_shared.spec.yml | 1366 +++ intrinsics_data/arm_intrinsics.json | 7507 +++++++++++++---- 5 files changed, 12558 insertions(+), 1708 deletions(-) diff --git a/crates/core_arch/src/aarch64/neon/generated.rs b/crates/core_arch/src/aarch64/neon/generated.rs index 0f92c6dec0..3f2d55a9fe 100644 --- a/crates/core_arch/src/aarch64/neon/generated.rs +++ b/crates/core_arch/src/aarch64/neon/generated.rs @@ -4604,6 +4604,36 @@ pub fn vcombine_f64(a: float64x1_t, b: float64x1_t) -> float64x2_t { } } #[doc = "Insert vector element from another vector element"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_bf16)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 3, LANE2 = 3))] +#[rustc_legacy_const_generics(1, 3)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +pub fn vcopy_lane_bf16( + a: bfloat16x4_t, + b: bfloat16x4_t, +) -> bfloat16x4_t { + unsafe { transmute(vcopy_lane_u16::(transmute(a), transmute(b))) } +} +#[doc = "Insert vector element from another vector element"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_bf16)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 7, LANE2 = 3))] +#[rustc_legacy_const_generics(1, 3)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +pub fn vcopyq_lane_bf16( + a: bfloat16x8_t, + b: bfloat16x4_t, +) -> bfloat16x8_t { + unsafe { transmute(vcopyq_lane_u16::(transmute(a), transmute(b))) } +} +#[doc = "Insert vector element from another vector element"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_f32)"] #[inline] #[target_feature(enable = "neon")] @@ -4811,6 +4841,36 @@ pub fn vcopy_lane_p64( b } #[doc = "Insert vector element from another vector element"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_bf16)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 3, LANE2 = 7))] +#[rustc_legacy_const_generics(1, 3)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +pub fn vcopy_laneq_bf16( + a: bfloat16x4_t, + b: bfloat16x8_t, +) -> bfloat16x4_t { + unsafe { transmute(vcopy_laneq_u16::(transmute(a), transmute(b))) } +} +#[doc = "Insert vector element from another vector element"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_bf16)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 7, LANE2 = 7))] +#[rustc_legacy_const_generics(1, 3)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +pub fn vcopyq_laneq_bf16( + a: bfloat16x8_t, + b: bfloat16x8_t, +) -> bfloat16x8_t { + unsafe { transmute(vcopyq_laneq_u16::(transmute(a), transmute(b))) } +} +#[doc = "Insert vector element from another vector element"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_f32)"] #[inline] #[target_feature(enable = "neon")] @@ -11411,6 +11471,50 @@ pub unsafe fn vldap1q_lane_p64(ptr: *const p64, src: poly64x2_t transmute(vldap1q_lane_s64::(ptr as *mut i64, transmute(src))) } #[doc = "Lookup table read with 2-bit indices"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[target_feature(enable = "neon,lut,bf16")] +#[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(luti2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +pub unsafe fn vluti2_lane_bf16(a: bfloat16x4_t, b: uint8x8_t) -> bfloat16x8_t { + static_assert!(LANE >= 0 && LANE <= 3); + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.vluti2.lane.v8bf16.v4bf16" + )] + fn _vluti2_lane_bf16(a: bfloat16x4_t, b: uint8x8_t, n: i32) -> bfloat16x8_t; + } + _vluti2_lane_bf16(a, b, LANE) +} +#[doc = "Lookup table read with 2-bit indices"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2q_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[target_feature(enable = "neon,lut,bf16")] +#[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(luti2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +pub unsafe fn vluti2q_lane_bf16(a: bfloat16x8_t, b: uint8x8_t) -> bfloat16x8_t { + static_assert!(LANE >= 0 && LANE <= 3); + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.vluti2.lane.v8bf16.v8bf16" + )] + fn _vluti2q_lane_bf16(a: bfloat16x8_t, b: uint8x8_t, n: i32) -> bfloat16x8_t; + } + _vluti2q_lane_bf16(a, b, LANE) +} +#[doc = "Lookup table read with 2-bit indices"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2_lane_f16)"] #[doc = "## Safety"] #[doc = " * Neon intrinsic unsafe"] @@ -11621,6 +11725,50 @@ pub unsafe fn vluti2q_lane_s16(a: int16x8_t, b: uint8x8_t) -> i _vluti2q_lane_s16(a, b, LANE) } #[doc = "Lookup table read with 2-bit indices"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2_laneq_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[target_feature(enable = "neon,lut,bf16")] +#[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(luti2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +pub unsafe fn vluti2_laneq_bf16(a: bfloat16x4_t, b: uint8x16_t) -> bfloat16x8_t { + static_assert!(LANE >= 0 && LANE <= 7); + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.vluti2.laneq.v8bf16.v4bf16" + )] + fn _vluti2_laneq_bf16(a: bfloat16x4_t, b: uint8x16_t, n: i32) -> bfloat16x8_t; + } + _vluti2_laneq_bf16(a, b, LANE) +} +#[doc = "Lookup table read with 2-bit indices"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2q_laneq_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[target_feature(enable = "neon,lut,bf16")] +#[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(luti2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +pub unsafe fn vluti2q_laneq_bf16(a: bfloat16x8_t, b: uint8x16_t) -> bfloat16x8_t { + static_assert!(LANE >= 0 && LANE <= 7); + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.vluti2.laneq.v8bf16.v8bf16" + )] + fn _vluti2q_laneq_bf16(a: bfloat16x8_t, b: uint8x16_t, n: i32) -> bfloat16x8_t; + } + _vluti2q_laneq_bf16(a, b, LANE) +} +#[doc = "Lookup table read with 2-bit indices"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2_laneq_f16)"] #[doc = "## Safety"] #[doc = " * Neon intrinsic unsafe"] @@ -11831,6 +11979,36 @@ pub unsafe fn vluti2q_laneq_s16(a: int16x8_t, b: uint8x16_t) - _vluti2q_laneq_s16(a, b, INDEX) } #[doc = "Lookup table read with 4-bit indices"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_lane_bf16_x2)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[target_feature(enable = "neon,lut,bf16")] +#[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(luti4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +pub unsafe fn vluti4q_lane_bf16_x2( + a: bfloat16x8x2_t, + b: uint8x8_t, +) -> bfloat16x8_t { + static_assert!(LANE >= 0 && LANE <= 1); + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.vluti4q.lane.x2.v8bf16" + )] + fn _vluti4q_lane_bf16_x2( + a0: bfloat16x8_t, + a1: bfloat16x8_t, + b: uint8x8_t, + n: i32, + ) -> bfloat16x8_t; + } + _vluti4q_lane_bf16_x2(a.0, a.1, b, LANE) +} +#[doc = "Lookup table read with 4-bit indices"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_lane_f16_x2)"] #[doc = "## Safety"] #[doc = " * Neon intrinsic unsafe"] @@ -11936,6 +12114,36 @@ pub unsafe fn vluti4q_lane_p8(a: poly8x16_t, b: uint8x8_t) -> p transmute(vluti4q_lane_s8::(transmute(a), b)) } #[doc = "Lookup table read with 4-bit indices"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_laneq_bf16_x2)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[target_feature(enable = "neon,lut,bf16")] +#[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(luti4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +pub unsafe fn vluti4q_laneq_bf16_x2( + a: bfloat16x8x2_t, + b: uint8x16_t, +) -> bfloat16x8_t { + static_assert!(LANE >= 0 && LANE <= 3); + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.vluti4q.laneq.x2.v8bf16" + )] + fn _vluti4q_laneq_bf16_x2( + a0: bfloat16x8_t, + a1: bfloat16x8_t, + b: uint8x16_t, + n: i32, + ) -> bfloat16x8_t; + } + _vluti4q_laneq_bf16_x2(a.0, a.1, b, LANE) +} +#[doc = "Lookup table read with 4-bit indices"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_laneq_f16_x2)"] #[doc = "## Safety"] #[doc = " * Neon intrinsic unsafe"] @@ -21187,6 +21395,54 @@ pub fn vrecpxh_f16(a: f16) -> f16 { unsafe { _vrecpxh_f16(a) } } #[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_f64)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[cfg(not(target_arch = "arm64ec"))] +#[cfg_attr(test, assert_instr(nop))] +pub fn vreinterpret_bf16_f64(a: float64x1_t) -> bfloat16x4_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_f64)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[cfg(not(target_arch = "arm64ec"))] +#[cfg_attr(test, assert_instr(nop))] +pub fn vreinterpretq_bf16_f64(a: float64x2_t) -> bfloat16x8_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_bf16)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[cfg(not(target_arch = "arm64ec"))] +#[cfg_attr(test, assert_instr(nop))] +pub fn vreinterpret_f64_bf16(a: bfloat16x4_t) -> float64x1_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_bf16)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[cfg(not(target_arch = "arm64ec"))] +#[cfg_attr(test, assert_instr(nop))] +pub fn vreinterpretq_f64_bf16(a: bfloat16x8_t) -> float64x2_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_f16)"] #[inline] #[target_feature(enable = "neon")] diff --git a/crates/core_arch/src/arm_shared/neon/generated.rs b/crates/core_arch/src/arm_shared/neon/generated.rs index 237f8fb413..684f4c5a79 100644 --- a/crates/core_arch/src/arm_shared/neon/generated.rs +++ b/crates/core_arch/src/arm_shared/neon/generated.rs @@ -3753,6 +3753,594 @@ pub fn vand_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { pub fn vandq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { unsafe { simd_and(a, b) } } +#[doc = "BFloat16 floating-point dot product (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfdot_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdot.bf16"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfdot) +)] +pub fn vbfdot_f32(a: float32x2_t, b: bfloat16x4_t, c: bfloat16x4_t) -> float32x2_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfdot.v2f32.v4bf16" + )] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.bfdot.v2f32.v4bf16")] + fn _vbfdot_f32(a: float32x2_t, b: bfloat16x4_t, c: bfloat16x4_t) -> float32x2_t; + } + unsafe { _vbfdot_f32(a, b, c) } +} +#[doc = "BFloat16 floating-point dot product (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfdotq_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdot.bf16"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfdot) +)] +pub fn vbfdotq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfdot.v4f32.v8bf16" + )] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.bfdot.v4f32.v8bf16")] + fn _vbfdotq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t; + } + unsafe { _vbfdotq_f32(a, b, c) } +} +#[doc = "BFloat16 floating-point dot product (vector, by element)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfdot_lane_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdot.bf16", LANE = 0))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfdot, LANE = 0) +)] +pub fn vbfdot_lane_f32( + a: float32x2_t, + b: bfloat16x4_t, + c: bfloat16x4_t, +) -> float32x2_t { + static_assert_uimm_bits!(LANE, 1); + let pairs: bfloat16x4_t = c; + unsafe { + let duped: bfloat16x4_t = transmute(vdup_lane_u32::(transmute(pairs))); + vbfdot_f32(a, b, duped) + } +} +#[doc = "BFloat16 floating-point dot product (vector, by element)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfdotq_lane_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdot.bf16", LANE = 0))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfdot, LANE = 0) +)] +pub fn vbfdotq_lane_f32( + a: float32x4_t, + b: bfloat16x8_t, + c: bfloat16x4_t, +) -> float32x4_t { + static_assert_uimm_bits!(LANE, 1); + let pairs: bfloat16x4_t = c; + unsafe { + let duped: bfloat16x8_t = transmute(vdupq_lane_u32::(transmute(pairs))); + vbfdotq_f32(a, b, duped) + } +} +#[doc = "BFloat16 floating-point dot product (vector, by element)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfdot_laneq_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdot.bf16", LANE = 0))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfdot, LANE = 0) +)] +pub fn vbfdot_laneq_f32( + a: float32x2_t, + b: bfloat16x4_t, + c: bfloat16x8_t, +) -> float32x2_t { + static_assert_uimm_bits!(LANE, 2); + let pairs: bfloat16x8_t = c; + unsafe { + let duped: bfloat16x4_t = transmute(vdup_laneq_u32::(transmute(pairs))); + vbfdot_f32(a, b, duped) + } +} +#[doc = "BFloat16 floating-point dot product (vector, by element)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfdotq_laneq_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdot.bf16", LANE = 0))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfdot, LANE = 0) +)] +pub fn vbfdotq_laneq_f32( + a: float32x4_t, + b: bfloat16x8_t, + c: bfloat16x8_t, +) -> float32x4_t { + static_assert_uimm_bits!(LANE, 2); + let pairs: bfloat16x8_t = c; + unsafe { + let duped: bfloat16x8_t = transmute(vdupq_laneq_u32::(transmute(pairs))); + vbfdotq_f32(a, b, duped) + } +} +#[doc = "BFloat16 floating-point widening multiply-add long (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmlalbq_f32)"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vfmab.bf16"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfmlalb) +)] +pub fn vbfmlalbq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfmlalb" + )] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.bfmlalb")] + fn _vbfmlalbq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t; + } + unsafe { _vbfmlalbq_f32(a, b, c) } +} +#[doc = "BFloat16 floating-point widening multiply-add long (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmlalbq_f32)"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vfmab.bf16"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfmlalb) +)] +pub fn vbfmlalbq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfmlalb" + )] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.bfmlalb")] + fn _vbfmlalbq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t; + } + unsafe { + let a: float32x4_t = simd_shuffle!(a, a, [3, 2, 1, 0]); + let b: bfloat16x8_t = simd_shuffle!(b, b, [7, 6, 5, 4, 3, 2, 1, 0]); + let c: bfloat16x8_t = simd_shuffle!(c, c, [7, 6, 5, 4, 3, 2, 1, 0]); + let ret_val: float32x4_t = _vbfmlalbq_f32(a, b, c); + simd_shuffle!(ret_val, ret_val, [3, 2, 1, 0]) + } +} +#[doc = "BFloat16 floating-point widening multiply-add long (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmlalbq_lane_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vfmab.bf16", LANE = 0))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfmlalb, LANE = 0) +)] +pub fn vbfmlalbq_lane_f32( + a: float32x4_t, + b: bfloat16x8_t, + c: bfloat16x4_t, +) -> float32x4_t { + static_assert_uimm_bits!(LANE, 2); + unsafe { + let duped: bfloat16x8_t = transmute(vdupq_lane_u16::(transmute(c))); + vbfmlalbq_f32(a, b, duped) + } +} +#[doc = "BFloat16 floating-point widening multiply-add long (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmlalbq_laneq_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vfmab.bf16", LANE = 0))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfmlalb, LANE = 0) +)] +pub fn vbfmlalbq_laneq_f32( + a: float32x4_t, + b: bfloat16x8_t, + c: bfloat16x8_t, +) -> float32x4_t { + static_assert_uimm_bits!(LANE, 3); + unsafe { + let duped: bfloat16x8_t = transmute(vdupq_laneq_u16::(transmute(c))); + vbfmlalbq_f32(a, b, duped) + } +} +#[doc = "BFloat16 floating-point widening multiply-add long (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmlaltq_f32)"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vfmat.bf16"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfmlalt) +)] +pub fn vbfmlaltq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfmlalt" + )] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.bfmlalt")] + fn _vbfmlaltq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t; + } + unsafe { _vbfmlaltq_f32(a, b, c) } +} +#[doc = "BFloat16 floating-point widening multiply-add long (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmlaltq_f32)"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vfmat.bf16"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfmlalt) +)] +pub fn vbfmlaltq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfmlalt" + )] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.bfmlalt")] + fn _vbfmlaltq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t; + } + unsafe { + let a: float32x4_t = simd_shuffle!(a, a, [3, 2, 1, 0]); + let b: bfloat16x8_t = simd_shuffle!(b, b, [7, 6, 5, 4, 3, 2, 1, 0]); + let c: bfloat16x8_t = simd_shuffle!(c, c, [7, 6, 5, 4, 3, 2, 1, 0]); + let ret_val: float32x4_t = _vbfmlaltq_f32(a, b, c); + simd_shuffle!(ret_val, ret_val, [3, 2, 1, 0]) + } +} +#[doc = "BFloat16 floating-point widening multiply-add long (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmlaltq_lane_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vfmat.bf16", LANE = 0))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfmlalt, LANE = 0) +)] +pub fn vbfmlaltq_lane_f32( + a: float32x4_t, + b: bfloat16x8_t, + c: bfloat16x4_t, +) -> float32x4_t { + static_assert_uimm_bits!(LANE, 2); + unsafe { + let duped: bfloat16x8_t = transmute(vdupq_lane_u16::(transmute(c))); + vbfmlaltq_f32(a, b, duped) + } +} +#[doc = "BFloat16 floating-point widening multiply-add long (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmlaltq_laneq_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vfmat.bf16", LANE = 0))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfmlalt, LANE = 0) +)] +pub fn vbfmlaltq_laneq_f32( + a: float32x4_t, + b: bfloat16x8_t, + c: bfloat16x8_t, +) -> float32x4_t { + static_assert_uimm_bits!(LANE, 3); + unsafe { + let duped: bfloat16x8_t = transmute(vdupq_laneq_u16::(transmute(c))); + vbfmlaltq_f32(a, b, duped) + } +} +#[doc = "BFloat16 floating-point matrix multiply-accumulate into 2x2 matrix"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmmlaq_f32)"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmmla.bf16"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfmmla) +)] +pub fn vbfmmlaq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfmmla" + )] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.bfmmla")] + fn _vbfmmlaq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t; + } + unsafe { _vbfmmlaq_f32(a, b, c) } +} +#[doc = "BFloat16 floating-point matrix multiply-accumulate into 2x2 matrix"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmmlaq_f32)"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmmla.bf16"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfmmla) +)] +pub fn vbfmmlaq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfmmla" + )] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.bfmmla")] + fn _vbfmmlaq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x4_t; + } + unsafe { + let a: float32x4_t = simd_shuffle!(a, a, [3, 2, 1, 0]); + let b: bfloat16x8_t = simd_shuffle!(b, b, [7, 6, 5, 4, 3, 2, 1, 0]); + let c: bfloat16x8_t = simd_shuffle!(c, c, [7, 6, 5, 4, 3, 2, 1, 0]); + let ret_val: float32x4_t = _vbfmmlaq_f32(a, b, c); + simd_shuffle!(ret_val, ret_val, [3, 2, 1, 0]) + } +} #[doc = "Vector bitwise bit clear."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbic_s16)"] #[inline] @@ -7758,6 +8346,59 @@ pub fn vcntq_p8(a: poly8x16_t) -> poly8x16_t { unsafe { transmute(vcntq_s8(transmute(a))) } } #[doc = "Join two smaller vectors into a single larger vector"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_bf16)"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub fn vcombine_bf16(a: bfloat16x4_t, b: bfloat16x4_t) -> bfloat16x8_t { + unsafe { simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]) } +} +#[doc = "Join two smaller vectors into a single larger vector"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_bf16)"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub fn vcombine_bf16(a: bfloat16x4_t, b: bfloat16x4_t) -> bfloat16x8_t { + unsafe { + let a: bfloat16x4_t = simd_shuffle!(a, a, [3, 2, 1, 0]); + let b: bfloat16x4_t = simd_shuffle!(b, b, [3, 2, 1, 0]); + let ret_val: bfloat16x8_t = simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]); + simd_shuffle!(ret_val, ret_val, [7, 6, 5, 4, 3, 2, 1, 0]) + } +} +#[doc = "Join two smaller vectors into a single larger vector"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_f16)"] #[inline] #[cfg(target_endian = "little")] @@ -8304,6 +8945,31 @@ pub fn vcombine_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x2_t { } } #[doc = "Insert vector element from another vector element"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vcreate_bf16(a: u64) -> bfloat16x4_t { + transmute(a) +} +#[doc = "Insert vector element from another vector element"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_f16)"] #[inline] #[target_feature(enable = "neon")] @@ -8578,6 +9244,48 @@ pub fn vcreate_p16(a: u64) -> poly16x4_t { pub fn vcreate_p64(a: u64) -> poly64x1_t { unsafe { transmute(a) } } +#[doc = "Floating-point convert from single-precision to BFloat16 format (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_bf16_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt.bf16.f32"))] +pub fn vcvt_bf16_f32(a: float32x4_t) -> bfloat16x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfp2bf.v4bf16")] + fn _vcvt_bf16_f32(a: float32x4_t) -> bfloat16x4_t; + } + unsafe { _vcvt_bf16_f32(a) } +} +#[doc = "Floating-point convert from single-precision to BFloat16 format (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_bf16_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(bfcvtn) +)] +pub fn vcvt_bf16_f32(a: float32x4_t) -> bfloat16x4_t { + unsafe { vget_low_bf16(vcvtq_low_bf16_f32(a)) } +} #[doc = "Floating-point convert to lower precision narrow"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f16_f32)"] #[inline] @@ -8689,6 +9397,32 @@ pub fn vcvt_f16_u16(a: uint16x4_t) -> float16x4_t { pub fn vcvtq_f16_u16(a: uint16x8_t) -> float16x8_t { unsafe { simd_cast(a) } } +#[doc = "Shift Left Long (by element size)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f32_bf16)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.i16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(shll) +)] +pub fn vcvt_f32_bf16(a: bfloat16x4_t) -> float32x4_t { + unsafe { + let half: uint16x4_t = transmute(a); + vreinterpretq_f32_u32(vshll_n_u16::<16>(half)) + } +} #[doc = "Floating-point convert to higher precision long"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f32_f16)"] #[inline] @@ -9584,6 +10318,371 @@ pub fn vcvtq_u32_f32(a: float32x4_t) -> uint32x4_t { } unsafe { _vcvtq_u32_f32(a) } } +#[doc = "Shift Left (immediate)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtah_f32_bf16)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub fn vcvtah_f32_bf16(a: bfloat16_t) -> f32 { + f32::from_bits((a.to_bits() as u32) << 16) +} +#[doc = "Floating-point convert from single-precision to BFloat16 format (scalar)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_bf16_f32)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvtb.bf16.f32"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + not(target_env = "msvc") + ), + assert_instr(bfcvt) +)] +pub fn vcvth_bf16_f32(a: f32) -> bfloat16_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtbfp2bf")] + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfcvt" + )] + fn _vcvth_bf16_f32(a: f32) -> bfloat16_t; + } + unsafe { _vcvth_bf16_f32(a) } +} +#[doc = "Floating-point convert from single-precision to BFloat16 format (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_high_bf16_f32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + target_endian = "little" + ), + assert_instr(bfcvtn2) +)] +#[cfg_attr( + all(test, target_arch = "aarch64", target_endian = "big"), + assert_instr(bfcvtn) +)] +pub unsafe fn vcvtq_high_bf16_f32(_inactive: bfloat16x8_t, a: float32x4_t) -> bfloat16x8_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfcvtn2" + )] + fn _vcvtq_high_bf16_f32(_inactive: bfloat16x8_t, a: float32x4_t) -> bfloat16x8_t; + } + _vcvtq_high_bf16_f32(_inactive, a) +} +#[doc = "Floating-point convert from single-precision to BFloat16 format (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_high_bf16_f32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + target_endian = "little" + ), + assert_instr(bfcvtn2) +)] +#[cfg_attr( + all(test, target_arch = "aarch64", target_endian = "big"), + assert_instr(bfcvtn) +)] +pub unsafe fn vcvtq_high_bf16_f32(_inactive: bfloat16x8_t, a: float32x4_t) -> bfloat16x8_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfcvtn2" + )] + fn _vcvtq_high_bf16_f32(_inactive: bfloat16x8_t, a: float32x4_t) -> bfloat16x8_t; + } + let _inactive: bfloat16x8_t = simd_shuffle!(_inactive, _inactive, [7, 6, 5, 4, 3, 2, 1, 0]); + let a: float32x4_t = simd_shuffle!(a, a, [3, 2, 1, 0]); + let ret_val: bfloat16x8_t = _vcvtq_high_bf16_f32(_inactive, a); + simd_shuffle!(ret_val, ret_val, [7, 6, 5, 4, 3, 2, 1, 0]) +} +#[doc = "Floating-point convert from single-precision to BFloat16 format (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_high_bf16_f32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt.bf16.f32"))] +pub unsafe fn vcvtq_high_bf16_f32(_inactive: bfloat16x8_t, a: float32x4_t) -> bfloat16x8_t { + let b: bfloat16x4_t = simd_shuffle!(_inactive, _inactive, [0, 1, 2, 3]); + let a: int32x4_t = transmute(a); + let a: int16x8_t = transmute(a); + let a: float32x4_t = transmute(a); + let a: bfloat16x4_t = vcvt_bf16_f32(a); + let vec: bfloat16x4_t = simd_shuffle!(a, a, [0, 1, 2, 3]); + simd_shuffle!(vec, b, [0, 1, 2, 3, 4, 5, 6, 7]) +} +#[doc = "Shift Left Long (by element size)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_high_f32_bf16)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.i16"))] +#[cfg_attr( + all( + test, + any(target_arch = "aarch64", target_arch = "arm64ec"), + target_endian = "little" + ), + assert_instr(shll2) +)] +#[cfg_attr( + all(test, target_arch = "aarch64", target_endian = "big"), + assert_instr(shll) +)] +pub fn vcvtq_high_f32_bf16(a: bfloat16x8_t) -> float32x4_t { + unsafe { + let half: uint16x4_t = vget_high_u16(transmute(a)); + vreinterpretq_f32_u32(vshll_n_u16::<16>(half)) + } +} +#[doc = "Floating-point convert from single-precision to BFloat16 format (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_low_bf16_f32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt.bf16.f32"))] +pub unsafe fn vcvtq_low_bf16_f32(a: float32x4_t) -> bfloat16x8_t { + let a: int32x4_t = transmute(a); + let a: int8x16_t = transmute(a); + let a: float32x4_t = transmute(a); + let vec: bfloat16x4_t = vcvt_bf16_f32(a); + let z: bfloat16x4_t = bfloat16x4_t([ + bfloat16_t::from_bits(0x0), + bfloat16_t::from_bits(0x0), + bfloat16_t::from_bits(0x0), + bfloat16_t::from_bits(0x0), + ]); + simd_shuffle!(z, vec, [0, 1, 2, 3, 4, 5, 6, 7]) +} +#[doc = "Floating-point convert from single-precision to BFloat16 format (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_low_bf16_f32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt.bf16.f32"))] +pub unsafe fn vcvtq_low_bf16_f32(a: float32x4_t) -> bfloat16x8_t { + let a: float32x4_t = simd_shuffle!(a, a, [3, 2, 1, 0]); + let a: int32x4_t = transmute(a); + let a: int8x16_t = transmute(a); + let a: float32x4_t = transmute(a); + let vec: bfloat16x4_t = vcvt_bf16_f32(a); + let z: bfloat16x4_t = bfloat16x4_t([ + bfloat16_t::from_bits(0x0), + bfloat16_t::from_bits(0x0), + bfloat16_t::from_bits(0x0), + bfloat16_t::from_bits(0x0), + ]); + let ret_val: bfloat16x8_t = simd_shuffle!(z, vec, [0, 1, 2, 3, 4, 5, 6, 7]); + simd_shuffle!(ret_val, ret_val, [7, 6, 5, 4, 3, 2, 1, 0]) +} +#[doc = "Floating-point convert from single-precision to BFloat16 format (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_low_bf16_f32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(bfcvtn) +)] +pub unsafe fn vcvtq_low_bf16_f32(a: float32x4_t) -> bfloat16x8_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfcvtn" + )] + fn _vcvtq_low_bf16_f32(a: float32x4_t) -> bfloat16x8_t; + } + _vcvtq_low_bf16_f32(a) +} +#[doc = "Floating-point convert from single-precision to BFloat16 format (vector)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_low_bf16_f32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(bfcvtn) +)] +pub unsafe fn vcvtq_low_bf16_f32(a: float32x4_t) -> bfloat16x8_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.bfcvtn" + )] + fn _vcvtq_low_bf16_f32(a: float32x4_t) -> bfloat16x8_t; + } + let a: float32x4_t = simd_shuffle!(a, a, [3, 2, 1, 0]); + let ret_val: bfloat16x8_t = _vcvtq_low_bf16_f32(a); + simd_shuffle!(ret_val, ret_val, [7, 6, 5, 4, 3, 2, 1, 0]) +} +#[doc = "Shift Left Long (by element size)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_low_f32_bf16)"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.i16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(shll) +)] +pub fn vcvtq_low_f32_bf16(a: bfloat16x8_t) -> float32x4_t { + unsafe { + let half: uint16x4_t = vget_low_u16(transmute(a)); + vreinterpretq_f32_u32(vshll_n_u16::<16>(half)) + } +} #[doc = "Dot product arithmetic (indexed)"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_lane_s32)"] #[inline] @@ -10052,6 +11151,34 @@ pub fn vdotq_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x16_t) -> uint32x4_t { simd_shuffle!(ret_val, ret_val, [3, 2, 1, 0]) } } +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vdup_lane_bf16(a: bfloat16x4_t) -> bfloat16x4_t { + static_assert_uimm_bits!(LANE, 2); + let c: uint16x4_t = vreinterpret_u16_bf16(a); + let duped: uint16x4_t = vdup_lane_u16::(c); + vreinterpret_bf16_u16(duped) +} #[doc = "Set all vector lanes to the same value"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_f16)"] #[inline] @@ -11158,6 +12285,118 @@ pub fn vdup_lane_u64(a: uint64x1_t) -> uint64x1_t { static_assert!(N == 0); a } +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vdup_laneq_bf16(a: bfloat16x8_t) -> bfloat16x4_t { + static_assert_uimm_bits!(LANE, 3); + simd_shuffle!(a, a, [LANE as u32; 4]) +} +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vdup_laneq_bf16(a: bfloat16x8_t) -> bfloat16x4_t { + static_assert_uimm_bits!(LANE, 3); + let a: bfloat16x8_t = simd_shuffle!(a, a, [7, 6, 5, 4, 3, 2, 1, 0]); + let ret_val: bfloat16x4_t = simd_shuffle!(a, a, [LANE as u32; 4]); + simd_shuffle!(ret_val, ret_val, [3, 2, 1, 0]) +} +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vdupq_laneq_bf16(a: bfloat16x8_t) -> bfloat16x8_t { + static_assert_uimm_bits!(LANE, 3); + simd_shuffle!(a, a, [LANE as u32; 8]) +} +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vdupq_laneq_bf16(a: bfloat16x8_t) -> bfloat16x8_t { + static_assert_uimm_bits!(LANE, 3); + let a: bfloat16x8_t = simd_shuffle!(a, a, [7, 6, 5, 4, 3, 2, 1, 0]); + let ret_val: bfloat16x8_t = simd_shuffle!(a, a, [LANE as u32; 8]); + simd_shuffle!(ret_val, ret_val, [7, 6, 5, 4, 3, 2, 1, 0]) +} #[doc = "Set all vector lanes to the same value"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_f16)"] #[inline] @@ -12270,6 +13509,31 @@ pub fn vdup_laneq_u64(a: uint64x2_t) -> uint64x1_t { static_assert_uimm_bits!(N, 1); unsafe { transmute(vgetq_lane_u64::(a)) } } +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vdup_n_bf16(a: bfloat16_t) -> bfloat16x4_t { + vreinterpret_bf16_u16(vdup_n_u16(a.to_bits())) +} #[doc = "Create a new vector with all lanes set to a value"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_f16)"] #[inline] @@ -12806,6 +14070,86 @@ fn vdup_n_f32_vfp4(value: f32) -> float32x2_t { fn vdupq_n_f32_vfp4(value: f32) -> float32x4_t { float32x4_t::splat(value) } +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vduph_lane_bf16(a: bfloat16x4_t) -> bfloat16_t { + static_assert_uimm_bits!(LANE, 2); + vget_lane_bf16::(a) +} +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_laneq_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vduph_laneq_bf16(a: bfloat16x8_t) -> bfloat16_t { + static_assert_uimm_bits!(LANE, 3); + vgetq_lane_bf16::(a) +} +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vdupq_lane_bf16(a: bfloat16x4_t) -> bfloat16x8_t { + static_assert_uimm_bits!(LANE, 2); + let c: uint16x4_t = vreinterpret_u16_bf16(a); + let duped: uint16x8_t = vdupq_lane_u16::(c); + vreinterpretq_bf16_u16(duped) +} #[doc = "Set all vector lanes to the same value"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_s64)"] #[inline] @@ -13012,6 +14356,31 @@ pub fn vdupq_laneq_u64(a: uint64x2_t) -> uint64x2_t { simd_shuffle!(ret_val, ret_val, [1, 0]) } } +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vdupq_n_bf16(a: bfloat16_t) -> bfloat16x8_t { + vreinterpretq_bf16_u16(vdupq_n_u16(a.to_bits())) +} #[doc = "Vector bitwise exclusive or (vector)"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_s8)"] #[inline] @@ -15191,6 +16560,60 @@ pub fn vfms_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t { pub fn vfmsq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t { vfmsq_f32(a, b, vdupq_n_f32_vfp4(c)) } +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vget_high_bf16(a: bfloat16x8_t) -> bfloat16x4_t { + simd_shuffle!(a, a, [4, 5, 6, 7]) +} +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vget_high_bf16(a: bfloat16x8_t) -> bfloat16x4_t { + let a: bfloat16x8_t = simd_shuffle!(a, a, [7, 6, 5, 4, 3, 2, 1, 0]); + let ret_val: bfloat16x4_t = simd_shuffle!(a, a, [4, 5, 6, 7]); + simd_shuffle!(ret_val, ret_val, [3, 2, 1, 0]) +} #[doc = "Duplicate vector element to vector"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_f16)"] #[inline] @@ -15999,6 +17422,61 @@ pub fn vget_high_u64(a: uint64x2_t) -> uint64x1_t { uint64x1_t([simd_extract!(a, 1)]) } } +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vget_lane_bf16(a: bfloat16x4_t) -> bfloat16_t { + static_assert_uimm_bits!(LANE, 2); + simd_extract!(a, LANE as u32) +} +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vget_lane_bf16(a: bfloat16x4_t) -> bfloat16_t { + static_assert_uimm_bits!(LANE, 2); + let a: bfloat16x4_t = simd_shuffle!(a, a, [3, 2, 1, 0]); + simd_extract!(a, LANE as u32) +} #[doc = "Duplicate vector element to scalar"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_f16)"] #[inline] @@ -17045,6 +18523,31 @@ pub fn vget_lane_u64(v: uint64x1_t) -> u64 { unsafe { simd_extract!(v, 0) } } #[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vget_low_bf16(a: bfloat16x8_t) -> bfloat16x4_t { + transmute(vget_low_u64(transmute(a))) +} +#[doc = "Duplicate vector element to vector or scalar"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_f32)"] #[inline] #[cfg(target_endian = "little")] @@ -17524,6 +19027,61 @@ pub fn vget_low_u64(a: uint64x2_t) -> uint64x1_t { uint64x1_t([simd_extract!(a, 0)]) } } +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vgetq_lane_bf16(a: bfloat16x8_t) -> bfloat16_t { + static_assert_uimm_bits!(LANE, 3); + simd_extract!(a, LANE as u32) +} +#[doc = "Duplicate vector element to vector or scalar"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vgetq_lane_bf16(a: bfloat16x8_t) -> bfloat16_t { + static_assert_uimm_bits!(LANE, 3); + let a: bfloat16x8_t = simd_shuffle!(a, a, [7, 6, 5, 4, 3, 2, 1, 0]); + simd_extract!(a, LANE as u32) +} #[doc = "Halving add"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_s8)"] #[inline] @@ -18220,6 +19778,436 @@ pub fn vhsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { } unsafe { _vhsubq_u32(a, b) } } +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vld1_bf16(ptr: *const bfloat16_t) -> bfloat16x4_t { + crate::ptr::read_unaligned(ptr.cast()) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vld1q_bf16(ptr: *const bfloat16_t) -> bfloat16x8_t { + crate::ptr::read_unaligned(ptr.cast()) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_bf16_x2)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1_bf16_x2(ptr: *const bfloat16_t) -> bfloat16x4x2_t { + transmute(vld1_u16_x2(ptr.cast())) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_bf16_x3)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1_bf16_x3(ptr: *const bfloat16_t) -> bfloat16x4x3_t { + transmute(vld1_u16_x3(ptr.cast())) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_bf16_x4)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1_bf16_x4(ptr: *const bfloat16_t) -> bfloat16x4x4_t { + transmute(vld1_u16_x4(ptr.cast())) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_bf16_x2)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1q_bf16_x2(ptr: *const bfloat16_t) -> bfloat16x8x2_t { + transmute(vld1q_u16_x2(ptr.cast())) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_bf16_x3)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1q_bf16_x3(ptr: *const bfloat16_t) -> bfloat16x8x3_t { + transmute(vld1q_u16_x3(ptr.cast())) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_bf16_x4)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1q_bf16_x4(ptr: *const bfloat16_t) -> bfloat16x8x4_t { + transmute(vld1q_u16_x4(ptr.cast())) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_bf16_x2)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1_bf16_x2(ptr: *const bfloat16_t) -> bfloat16x4x2_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld1x2.v4bf16.p0" + )] + fn _vld1_bf16_x2(ptr: *const bfloat16_t) -> bfloat16x4x2_t; + } + _vld1_bf16_x2(ptr) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_bf16_x3)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1_bf16_x3(ptr: *const bfloat16_t) -> bfloat16x4x3_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld1x3.v4bf16.p0" + )] + fn _vld1_bf16_x3(ptr: *const bfloat16_t) -> bfloat16x4x3_t; + } + _vld1_bf16_x3(ptr) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_bf16_x4)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1_bf16_x4(ptr: *const bfloat16_t) -> bfloat16x4x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld1x4.v4bf16.p0" + )] + fn _vld1_bf16_x4(ptr: *const bfloat16_t) -> bfloat16x4x4_t; + } + _vld1_bf16_x4(ptr) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_bf16_x2)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1q_bf16_x2(ptr: *const bfloat16_t) -> bfloat16x8x2_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld1x2.v8bf16.p0" + )] + fn _vld1q_bf16_x2(ptr: *const bfloat16_t) -> bfloat16x8x2_t; + } + _vld1q_bf16_x2(ptr) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_bf16_x3)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1q_bf16_x3(ptr: *const bfloat16_t) -> bfloat16x8x3_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld1x3.v8bf16.p0" + )] + fn _vld1q_bf16_x3(ptr: *const bfloat16_t) -> bfloat16x8x3_t; + } + _vld1q_bf16_x3(ptr) +} +#[doc = "Load multiple single-element structures to one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_bf16_x4)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld1) +)] +pub unsafe fn vld1q_bf16_x4(ptr: *const bfloat16_t) -> bfloat16x8x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld1x4.v8bf16.p0" + )] + fn _vld1q_bf16_x4(ptr: *const bfloat16_t) -> bfloat16x8x4_t; + } + _vld1q_bf16_x4(ptr) +} +#[doc = "Load one single-element structure and Replicate to all lanes (of one register)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vld1_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x4_t { + transmute(vld1_dup_u16(ptr.cast())) +} +#[doc = "Load one single-element structure and Replicate to all lanes (of one register)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vld1q_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x8_t { + transmute(vld1q_dup_u16(ptr.cast())) +} #[doc = "Load one single-element structure and replicate to all lanes of one register"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_f16)"] #[doc = "## Safety"] @@ -19290,6 +21278,62 @@ pub unsafe fn vld1q_f32_x4(a: *const f32) -> float32x4x4_t { crate::ptr::read_unaligned(a.cast()) } #[doc = "Load one single-element structure to one lane of one register"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vld1_lane_bf16( + ptr: *const bfloat16_t, + src: bfloat16x4_t, +) -> bfloat16x4_t { + transmute(vld1_lane_u16::(ptr.cast(), transmute(src))) +} +#[doc = "Load one single-element structure to one lane of one register"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vld1q_lane_bf16( + ptr: *const bfloat16_t, + src: bfloat16x8_t, +) -> bfloat16x8_t { + transmute(vld1q_lane_u16::(ptr.cast(), transmute(src))) +} +#[doc = "Load one single-element structure to one lane of one register"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_f16)"] #[doc = "## Safety"] #[doc = " * Neon intrinsic unsafe"] @@ -21751,6 +23795,486 @@ pub unsafe fn vld1q_dup_p64(ptr: *const p64) -> poly64x2_t { let x = vld1q_lane_p64::<0>(ptr, transmute(u64x2::splat(0))); simd_shuffle!(x, x, [0, 0]) } +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld2.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld2) +)] +pub unsafe fn vld2_bf16(ptr: *const bfloat16_t) -> bfloat16x4x2_t { + transmute(vld2_u16(ptr.cast())) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld2.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld2) +)] +pub unsafe fn vld2q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x2_t { + transmute(vld2q_u16(ptr.cast())) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld3.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld3) +)] +pub unsafe fn vld3_bf16(ptr: *const bfloat16_t) -> bfloat16x4x3_t { + transmute(vld3_u16(ptr.cast())) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld3.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld3) +)] +pub unsafe fn vld3q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x3_t { + transmute(vld3q_u16(ptr.cast())) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld4.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld4) +)] +pub unsafe fn vld4_bf16(ptr: *const bfloat16_t) -> bfloat16x4x4_t { + transmute(vld4_u16(ptr.cast())) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(target_arch = "arm")] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld4.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld4) +)] +pub unsafe fn vld4q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x4_t { + transmute(vld4q_u16(ptr.cast())) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld2.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld2) +)] +pub unsafe fn vld2_bf16(ptr: *const bfloat16_t) -> bfloat16x4x2_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld2.v4bf16.p0" + )] + fn _vld2_bf16(ptr: *const bfloat16_t) -> bfloat16x4x2_t; + } + _vld2_bf16(ptr) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld2.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld2) +)] +pub unsafe fn vld2q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x2_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld2.v8bf16.p0" + )] + fn _vld2q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x2_t; + } + _vld2q_bf16(ptr) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld3.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld3) +)] +pub unsafe fn vld3_bf16(ptr: *const bfloat16_t) -> bfloat16x4x3_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld3.v4bf16.p0" + )] + fn _vld3_bf16(ptr: *const bfloat16_t) -> bfloat16x4x3_t; + } + _vld3_bf16(ptr) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld3.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld3) +)] +pub unsafe fn vld3q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x3_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld3.v8bf16.p0" + )] + fn _vld3q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x3_t; + } + _vld3q_bf16(ptr) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld4.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld4) +)] +pub unsafe fn vld4_bf16(ptr: *const bfloat16_t) -> bfloat16x4x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld4.v4bf16.p0" + )] + fn _vld4_bf16(ptr: *const bfloat16_t) -> bfloat16x4x4_t; + } + _vld4_bf16(ptr) +} +#[doc = "Load multiple 2-element structures to two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg(not(target_arch = "arm"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld4.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld4) +)] +pub unsafe fn vld4q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x4_t { + unsafe extern "llvm-intrinsic" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.neon.ld4.v8bf16.p0" + )] + fn _vld4q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x4_t; + } + _vld4q_bf16(ptr) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld2.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld2) +)] +pub unsafe fn vld2_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x4x2_t { + transmute(vld2_dup_u16(ptr.cast())) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld2.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld2) +)] +pub unsafe fn vld2q_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x8x2_t { + transmute(vld2q_dup_u16(ptr.cast())) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld3.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld3) +)] +pub unsafe fn vld3_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x4x3_t { + transmute(vld3_dup_u16(ptr.cast())) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld3.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld3) +)] +pub unsafe fn vld3q_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x8x3_t { + transmute(vld3q_dup_u16(ptr.cast())) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld4.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld4) +)] +pub unsafe fn vld4_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x4x4_t { + transmute(vld4_dup_u16(ptr.cast())) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld4.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld4) +)] +pub unsafe fn vld4q_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x8x4_t { + transmute(vld4q_dup_u16(ptr.cast())) +} #[doc = "Load single 2-element structure and replicate to all lanes of two registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_f16)"] #[doc = "## Safety"] @@ -22794,6 +25318,62 @@ pub unsafe fn vld2q_s32(a: *const i32) -> int32x4x2_t { } _vld2q_s32(a as _) } +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld2.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld2, LANE = 2) +)] +pub unsafe fn vld2_lane_bf16( + ptr: *const bfloat16_t, + src: bfloat16x4x2_t, +) -> bfloat16x4x2_t { + transmute(vld2_lane_u16::(ptr.cast(), transmute(src))) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld2.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld2, LANE = 2) +)] +pub unsafe fn vld2q_lane_bf16( + ptr: *const bfloat16_t, + src: bfloat16x8x2_t, +) -> bfloat16x8x2_t { + transmute(vld2q_lane_u16::(ptr.cast(), transmute(src))) +} #[doc = "Load multiple 2-element structures to two registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_f16)"] #[doc = "## Safety"] @@ -24699,6 +27279,62 @@ pub unsafe fn vld3q_s32(a: *const i32) -> int32x4x3_t { } _vld3q_s32(a as *const i8, 4) } +#[doc = "Load single 3-element structure to one lane of three registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld3.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld3, LANE = 2) +)] +pub unsafe fn vld3_lane_bf16( + ptr: *const bfloat16_t, + src: bfloat16x4x3_t, +) -> bfloat16x4x3_t { + transmute(vld3_lane_u16::(ptr.cast(), transmute(src))) +} +#[doc = "Load single 3-element structure to one lane of three registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld3.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld3, LANE = 2) +)] +pub unsafe fn vld3q_lane_bf16( + ptr: *const bfloat16_t, + src: bfloat16x8x3_t, +) -> bfloat16x8x3_t { + transmute(vld3q_lane_u16::(ptr.cast(), transmute(src))) +} #[doc = "Load multiple 3-element structures to two registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_f16)"] #[doc = "## Safety"] @@ -26650,6 +29286,62 @@ pub unsafe fn vld4q_s32(a: *const i32) -> int32x4x4_t { } _vld4q_s32(a as *const i8, 4) } +#[doc = "Load single 4-element structure and Replicate to all lanes of four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld4.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld4, LANE = 2) +)] +pub unsafe fn vld4_lane_bf16( + ptr: *const bfloat16_t, + src: bfloat16x4x4_t, +) -> bfloat16x4x4_t { + transmute(vld4_lane_u16::(ptr.cast(), transmute(src))) +} +#[doc = "Load single 4-element structure and Replicate to all lanes of four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld4.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(ld4, LANE = 2) +)] +pub unsafe fn vld4q_lane_bf16( + ptr: *const bfloat16_t, + src: bfloat16x8x4_t, +) -> bfloat16x8x4_t { + transmute(vld4q_lane_u16::(ptr.cast(), transmute(src))) +} #[doc = "Load multiple 4-element structures to two registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_f16)"] #[doc = "## Safety"] @@ -42264,293 +44956,657 @@ pub fn vrecpsq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { unsafe { _vrecpsq_f32(a, b) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_f16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_bf16_f16(a: float16x4_t) -> bfloat16x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_f32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_f32_f16(a: float16x4_t) -> float32x2_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_bf16_f32(a: float32x2_t) -> bfloat16x4_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_p16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_bf16_p16(a: poly16x4_t) -> bfloat16x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_p64)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_s8_f16(a: float16x4_t) -> int8x8_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_bf16_p64(a: poly64x1_t) -> bfloat16x4_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_p8)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_bf16_p8(a: poly8x8_t) -> bfloat16x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_s16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_s32_f16(a: float16x4_t) -> int32x2_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_bf16_s16(a: int16x4_t) -> bfloat16x4_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_s32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_bf16_s32(a: int32x2_t) -> bfloat16x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_s64)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_s64_f16(a: float16x4_t) -> int64x1_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_bf16_s64(a: int64x1_t) -> bfloat16x4_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_s8)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_bf16_s8(a: int8x8_t) -> bfloat16x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_u16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_u8_f16(a: float16x4_t) -> uint8x8_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_bf16_u16(a: uint16x4_t) -> bfloat16x4_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_u32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_bf16_u32(a: uint32x2_t) -> bfloat16x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_u64)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_u32_f16(a: float16x4_t) -> uint32x2_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_bf16_u64(a: uint64x1_t) -> bfloat16x4_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_bf16_u8)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_bf16_u8(a: uint8x8_t) -> bfloat16x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_u64_f16(a: float16x4_t) -> uint64x1_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_f16_bf16(a: bfloat16x4_t) -> float16x4_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_f32_bf16(a: bfloat16x4_t) -> float32x2_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_p8_f16(a: float16x4_t) -> poly8x8_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_p16_bf16(a: bfloat16x4_t) -> poly16x4_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_p64_bf16(a: bfloat16x4_t) -> poly64x1_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_f32_f16(a: float16x8_t) -> float32x4_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_p8_bf16(a: bfloat16x4_t) -> poly8x8_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_s16_bf16(a: bfloat16x4_t) -> int16x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_s8_f16(a: float16x8_t) -> int8x16_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_s32_bf16(a: bfloat16x4_t) -> int32x2_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_s64_bf16(a: bfloat16x4_t) -> int64x1_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_s32_f16(a: float16x8_t) -> int32x4_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_s8_bf16(a: bfloat16x4_t) -> int8x8_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_u16_bf16(a: bfloat16x4_t) -> uint16x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_s64_f16(a: float16x8_t) -> int64x2_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_u32_bf16(a: bfloat16x4_t) -> uint32x2_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(nop) )] +pub unsafe fn vreinterpret_u64_bf16(a: bfloat16x4_t) -> uint64x1_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - stable(feature = "stdarch_neon_fp16", since = "1.94.0") + unstable(feature = "stdarch_neon_bf16", issue = "none") )] #[cfg_attr( target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] -#[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_u8_f16(a: float16x8_t) -> uint8x16_t { - unsafe { transmute(a) } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpret_u8_bf16(a: bfloat16x4_t) -> uint8x8_t { + transmute(a) } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42568,11 +45624,11 @@ pub fn vreinterpretq_u8_f16(a: float16x8_t) -> uint8x16_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_u32_f16(a: float16x8_t) -> uint32x4_t { +pub fn vreinterpret_f32_f16(a: float16x4_t) -> float32x2_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42590,11 +45646,11 @@ pub fn vreinterpretq_u32_f16(a: float16x8_t) -> uint32x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_u64_f16(a: float16x8_t) -> uint64x2_t { +pub fn vreinterpret_s8_f16(a: float16x4_t) -> int8x8_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_f16)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42612,11 +45668,11 @@ pub fn vreinterpretq_u64_f16(a: float16x8_t) -> uint64x2_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_p8_f16(a: float16x8_t) -> poly8x16_t { +pub fn vreinterpret_s32_f16(a: float16x4_t) -> int32x2_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_f32)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42634,11 +45690,11 @@ pub fn vreinterpretq_p8_f16(a: float16x8_t) -> poly8x16_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_f16_f32(a: float32x2_t) -> float16x4_t { +pub fn vreinterpret_s64_f16(a: float16x4_t) -> int64x1_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_f32)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42656,11 +45712,11 @@ pub fn vreinterpret_f16_f32(a: float32x2_t) -> float16x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_f16_f32(a: float32x4_t) -> float16x8_t { +pub fn vreinterpret_u8_f16(a: float16x4_t) -> uint8x8_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s8)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42678,11 +45734,11 @@ pub fn vreinterpretq_f16_f32(a: float32x4_t) -> float16x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_f16_s8(a: int8x8_t) -> float16x4_t { +pub fn vreinterpret_u32_f16(a: float16x4_t) -> uint32x2_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s8)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42700,11 +45756,11 @@ pub fn vreinterpret_f16_s8(a: int8x8_t) -> float16x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_f16_s8(a: int8x16_t) -> float16x8_t { +pub fn vreinterpret_u64_f16(a: float16x4_t) -> uint64x1_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s32)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42722,11 +45778,11 @@ pub fn vreinterpretq_f16_s8(a: int8x16_t) -> float16x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_f16_s32(a: int32x2_t) -> float16x4_t { +pub fn vreinterpret_p8_f16(a: float16x4_t) -> poly8x8_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s32)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42744,11 +45800,11 @@ pub fn vreinterpret_f16_s32(a: int32x2_t) -> float16x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_f16_s32(a: int32x4_t) -> float16x8_t { +pub fn vreinterpretq_f32_f16(a: float16x8_t) -> float32x4_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s64)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42766,11 +45822,11 @@ pub fn vreinterpretq_f16_s32(a: int32x4_t) -> float16x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_f16_s64(a: int64x1_t) -> float16x4_t { +pub fn vreinterpretq_s8_f16(a: float16x8_t) -> int8x16_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s64)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42788,11 +45844,11 @@ pub fn vreinterpret_f16_s64(a: int64x1_t) -> float16x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_f16_s64(a: int64x2_t) -> float16x8_t { +pub fn vreinterpretq_s32_f16(a: float16x8_t) -> int32x4_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u8)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42810,11 +45866,11 @@ pub fn vreinterpretq_f16_s64(a: int64x2_t) -> float16x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_f16_u8(a: uint8x8_t) -> float16x4_t { +pub fn vreinterpretq_s64_f16(a: float16x8_t) -> int64x2_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u8)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42832,11 +45888,11 @@ pub fn vreinterpret_f16_u8(a: uint8x8_t) -> float16x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpretq_f16_u8(a: uint8x16_t) -> float16x8_t { +pub fn vreinterpretq_u8_f16(a: float16x8_t) -> uint8x16_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u32)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_f16)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -42854,11 +45910,297 @@ pub fn vreinterpretq_f16_u8(a: uint8x16_t) -> float16x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] #[cfg(not(target_arch = "arm64ec"))] -pub fn vreinterpret_f16_u32(a: uint32x2_t) -> float16x4_t { +pub fn vreinterpretq_u32_f16(a: float16x8_t) -> uint32x4_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u32)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_f16)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpretq_u64_f16(a: float16x8_t) -> uint64x2_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_f16)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpretq_p8_f16(a: float16x8_t) -> poly8x16_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_f32)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpret_f16_f32(a: float32x2_t) -> float16x4_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_f32)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpretq_f16_f32(a: float32x4_t) -> float16x8_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s8)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpret_f16_s8(a: int8x8_t) -> float16x4_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s8)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpretq_f16_s8(a: int8x16_t) -> float16x8_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s32)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpret_f16_s32(a: int32x2_t) -> float16x4_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s32)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpretq_f16_s32(a: int32x4_t) -> float16x8_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s64)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpret_f16_s64(a: int64x1_t) -> float16x4_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s64)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpretq_f16_s64(a: int64x2_t) -> float16x8_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u8)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpret_f16_u8(a: uint8x8_t) -> float16x4_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u8)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpretq_f16_u8(a: uint8x16_t) -> float16x8_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u32)"] +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +#[cfg_attr( + not(target_arch = "arm"), + stable(feature = "stdarch_neon_fp16", since = "1.94.0") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg(not(target_arch = "arm64ec"))] +pub fn vreinterpret_f16_u32(a: uint32x2_t) -> float16x4_t { + unsafe { transmute(a) } +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u32)"] #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -44072,6 +47414,706 @@ pub fn vreinterpretq_u16_p16(a: poly16x8_t) -> uint16x8_t { unsafe { transmute(a) } } #[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_f16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_f16(a: float16x8_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_f32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_f32(a: float32x4_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_p128)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_p128(a: p128) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_p16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_p16(a: poly16x8_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_p64)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_p64(a: poly64x2_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_p8)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_p8(a: poly8x16_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_s16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_s16(a: int16x8_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_s32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_s32(a: int32x4_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_s64)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_s64(a: int64x2_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_s8)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_s8(a: int8x16_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_u16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_u16(a: uint16x8_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_u32)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_u32(a: uint32x4_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_u64)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_u64(a: uint64x2_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_bf16_u8)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_bf16_u8(a: uint8x16_t) -> bfloat16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_f16_bf16(a: bfloat16x8_t) -> float16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_f32_bf16(a: bfloat16x8_t) -> float32x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_p128_bf16(a: bfloat16x8_t) -> p128 { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_p16_bf16(a: bfloat16x8_t) -> poly16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_p64_bf16(a: bfloat16x8_t) -> poly64x2_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_p8_bf16(a: bfloat16x8_t) -> poly8x16_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_s16_bf16(a: bfloat16x8_t) -> int16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_s32_bf16(a: bfloat16x8_t) -> int32x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_s64_bf16(a: bfloat16x8_t) -> int64x2_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_s8_bf16(a: bfloat16x8_t) -> int8x16_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_u16_bf16(a: bfloat16x8_t) -> uint16x8_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_u32_bf16(a: bfloat16x8_t) -> uint32x4_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_u64_bf16(a: bfloat16x8_t) -> uint64x2_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop) +)] +pub unsafe fn vreinterpretq_u8_bf16(a: bfloat16x8_t) -> uint8x16_t { + transmute(a) +} +#[doc = "Vector reinterpret cast operation"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_p128)"] #[inline] #[target_feature(enable = "neon")] @@ -53322,6 +57364,118 @@ pub fn vrsubhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t { unsafe { transmute(vrsubhn_s64(transmute(a), transmute(b))) } } #[doc = "Insert vector element from another vector element"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vset_lane_bf16(a: bfloat16_t, v: bfloat16x4_t) -> bfloat16x4_t { + static_assert_uimm_bits!(LANE, 2); + simd_insert!(v, LANE as u32, a) +} +#[doc = "Insert vector element from another vector element"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vset_lane_bf16(a: bfloat16_t, v: bfloat16x4_t) -> bfloat16x4_t { + static_assert_uimm_bits!(LANE, 2); + let v: bfloat16x4_t = simd_shuffle!(v, v, [3, 2, 1, 0]); + let ret_val: bfloat16x4_t = simd_insert!(v, LANE as u32, a); + simd_shuffle!(ret_val, ret_val, [3, 2, 1, 0]) +} +#[doc = "Insert vector element from another vector element"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "little")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vsetq_lane_bf16(a: bfloat16_t, v: bfloat16x8_t) -> bfloat16x8_t { + static_assert_uimm_bits!(LANE, 3); + simd_insert!(v, LANE as u32, a) +} +#[doc = "Insert vector element from another vector element"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(target_endian = "big")] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vsetq_lane_bf16(a: bfloat16_t, v: bfloat16x8_t) -> bfloat16x8_t { + static_assert_uimm_bits!(LANE, 3); + let v: bfloat16x8_t = simd_shuffle!(v, v, [7, 6, 5, 4, 3, 2, 1, 0]); + let ret_val: bfloat16x8_t = simd_insert!(v, LANE as u32, a); + simd_shuffle!(ret_val, ret_val, [7, 6, 5, 4, 3, 2, 1, 0]) +} +#[doc = "Insert vector element from another vector element"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_f16)"] #[inline] #[cfg(target_endian = "little")] @@ -57733,6 +61887,206 @@ pub fn vsriq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { static_assert!(1 <= N && N <= 16); unsafe { transmute(vshiftrins_v8i16::(transmute(a), transmute(b))) } } +#[doc = "Store multiple single-element structures from one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(str) +)] +pub unsafe fn vst1_bf16(ptr: *mut bfloat16_t, val: bfloat16x4_t) { + crate::ptr::write_unaligned(ptr.cast(), val) +} +#[doc = "Store multiple single-element structures from one, two, three, or four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst1.8"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(str) +)] +pub unsafe fn vst1q_bf16(ptr: *mut bfloat16_t, val: bfloat16x8_t) { + crate::ptr::write_unaligned(ptr.cast(), val) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_bf16_x2)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst1.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st1) +)] +pub unsafe fn vst1_bf16_x2(ptr: *mut bfloat16_t, val: bfloat16x4x2_t) { + vst1_u16_x2(ptr.cast(), transmute(val)) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_bf16_x3)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst1.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st1) +)] +pub unsafe fn vst1_bf16_x3(ptr: *mut bfloat16_t, val: bfloat16x4x3_t) { + vst1_u16_x3(ptr.cast(), transmute(val)) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_bf16_x4)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst1.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st1) +)] +pub unsafe fn vst1_bf16_x4(ptr: *mut bfloat16_t, val: bfloat16x4x4_t) { + vst1_u16_x4(ptr.cast(), transmute(val)) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_bf16_x2)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst1.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st1) +)] +pub unsafe fn vst1q_bf16_x2(ptr: *mut bfloat16_t, val: bfloat16x8x2_t) { + vst1q_u16_x2(ptr.cast(), transmute(val)) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_bf16_x3)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst1.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st1) +)] +pub unsafe fn vst1q_bf16_x3(ptr: *mut bfloat16_t, val: bfloat16x8x3_t) { + vst1q_u16_x3(ptr.cast(), transmute(val)) +} +#[doc = "Load single 2-element structure and Replicate to all lanes of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_bf16_x4)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst1.16"))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st1) +)] +pub unsafe fn vst1q_bf16_x4(ptr: *mut bfloat16_t, val: bfloat16x8x4_t) { + vst1q_u16_x4(ptr.cast(), transmute(val)) +} #[doc = "Store multiple single-element structures from one, two, three, or four registers."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f16)"] #[doc = "## Safety"] @@ -58561,6 +62915,56 @@ pub unsafe fn vst1q_f32_x4(a: *mut f32, b: float32x4x4_t) { } _vst1q_f32_x4(b.0, b.1, b.2, b.3, a) } +#[doc = "Store a single-element structure from one lane of one register"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vst1_lane_bf16(a: *mut bfloat16_t, v: bfloat16x4_t) { + vst1_lane_u16::(a.cast(), transmute(v)) +} +#[doc = "Store a single-element structure from one lane of one register"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 2) +)] +pub unsafe fn vst1q_lane_bf16(a: *mut bfloat16_t, v: bfloat16x8_t) { + vst1q_lane_u16::(a.cast(), transmute(v)) +} #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_f16)"] #[doc = "## Safety"] @@ -61184,6 +65588,56 @@ pub unsafe fn vst1q_lane_p64(a: *mut p64, b: poly64x2_t) { crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple 2-element structures from two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 0) +)] +pub unsafe fn vst2_bf16(a: *mut bfloat16_t, b: bfloat16x4x2_t) { + vst2_u16(a.cast(), transmute(b)) +} +#[doc = "Store multiple 2-element structures from two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 0) +)] +pub unsafe fn vst2q_bf16(a: *mut bfloat16_t, b: bfloat16x8x2_t) { + vst2q_u16(a.cast(), transmute(b)) +} +#[doc = "Store multiple 2-element structures from two registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f16)"] #[doc = "## Safety"] #[doc = " * Neon intrinsic unsafe"] @@ -61487,6 +65941,56 @@ pub unsafe fn vst2q_s32(a: *mut i32, b: int32x4x2_t) { } _vst2q_s32(a as _, b.0, b.1, 4) } +#[doc = "Store single 2-element structure from one lane of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst2.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st2, LANE = 2) +)] +pub unsafe fn vst2_lane_bf16(a: *mut bfloat16_t, v: bfloat16x4x2_t) { + vst2_lane_u16::(a.cast(), transmute(v)) +} +#[doc = "Store single 2-element structure from one lane of two registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst2.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st2, LANE = 2) +)] +pub unsafe fn vst2q_lane_bf16(a: *mut bfloat16_t, v: bfloat16x8x2_t) { + vst2q_lane_u16::(a.cast(), transmute(v)) +} #[doc = "Store multiple 2-element structures from two registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_f16)"] #[doc = "## Safety"] @@ -62349,6 +66853,56 @@ pub unsafe fn vst2q_p16(a: *mut p16, b: poly16x8x2_t) { vst2q_s16(transmute(a), transmute(b)) } #[doc = "Store multiple 3-element structures from three registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 0) +)] +pub unsafe fn vst3_bf16(a: *mut bfloat16_t, b: bfloat16x4x3_t) { + vst3_u16(a.cast(), transmute(b)) +} +#[doc = "Store multiple 3-element structures from three registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 0) +)] +pub unsafe fn vst3q_bf16(a: *mut bfloat16_t, b: bfloat16x8x3_t) { + vst3q_u16(a.cast(), transmute(b)) +} +#[doc = "Store multiple 3-element structures from three registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_f16)"] #[doc = "## Safety"] #[doc = " * Neon intrinsic unsafe"] @@ -62620,6 +67174,56 @@ pub unsafe fn vst3_s32(a: *mut i32, b: int32x2x3_t) { pub unsafe fn vst3q_s32(a: *mut i32, b: int32x4x3_t) { crate::core_arch::macros::interleaving_store!(i32, 4, 3, a, b) } +#[doc = "Store single 3-element structure from one lane of three registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst3.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st3, LANE = 2) +)] +pub unsafe fn vst3_lane_bf16(a: *mut bfloat16_t, v: bfloat16x4x3_t) { + vst3_lane_u16::(a.cast(), transmute(v)) +} +#[doc = "Store single 3-element structure from one lane of three registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst3.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st3, LANE = 2) +)] +pub unsafe fn vst3q_lane_bf16(a: *mut bfloat16_t, v: bfloat16x8x3_t) { + vst3q_lane_u16::(a.cast(), transmute(v)) +} #[doc = "Store multiple 3-element structures from three registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_f16)"] #[doc = "## Safety"] @@ -63538,6 +68142,56 @@ pub unsafe fn vst3q_p16(a: *mut p16, b: poly16x8x3_t) { vst3q_s16(transmute(a), transmute(b)) } #[doc = "Store multiple 4-element structures from four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 0) +)] +pub unsafe fn vst4_bf16(a: *mut bfloat16_t, b: bfloat16x4x4_t) { + vst4_u16(a.cast(), transmute(b)) +} +#[doc = "Store multiple 4-element structures from four registers"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(nop, LANE = 0) +)] +pub unsafe fn vst4q_bf16(a: *mut bfloat16_t, b: bfloat16x8x4_t) { + vst4q_u16(a.cast(), transmute(b)) +} +#[doc = "Store multiple 4-element structures from four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_f16)"] #[doc = "## Safety"] #[doc = " * Neon intrinsic unsafe"] @@ -63904,6 +68558,56 @@ pub unsafe fn vst4_s32(a: *mut i32, b: int32x2x4_t) { pub unsafe fn vst4q_s32(a: *mut i32, b: int32x4x4_t) { crate::core_arch::macros::interleaving_store!(i32, 4, 4, a, b) } +#[doc = "Store single 4-element structure from one lane of four registerss"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst4.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st4, LANE = 2) +)] +pub unsafe fn vst4_lane_bf16(a: *mut bfloat16_t, v: bfloat16x4x4_t) { + vst4_lane_u16::(a.cast(), transmute(v)) +} +#[doc = "Store single 4-element structure from one lane of four registerss"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_bf16)"] +#[doc = "## Safety"] +#[doc = " * Neon intrinsic unsafe"] +#[inline] +#[cfg(any(target_has_reliable_f16b, doc))] +#[doc(cfg(target_has_reliable_f16b))] +#[target_feature(enable = "neon,bf16")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr( + not(target_arch = "arm"), + unstable(feature = "stdarch_neon_bf16", issue = "none") +)] +#[cfg_attr( + target_arch = "arm", + unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") +)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vst4.16", LANE = 2))] +#[cfg_attr( + all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), + assert_instr(st4, LANE = 2) +)] +pub unsafe fn vst4q_lane_bf16(a: *mut bfloat16_t, v: bfloat16x8x4_t) { + vst4q_lane_u16::(a.cast(), transmute(v)) +} #[doc = "Store multiple 4-element structures from four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_f16)"] #[doc = "## Safety"] diff --git a/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml b/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml index 0b52f2701f..a1c955bc5c 100644 --- a/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml +++ b/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml @@ -41,6 +41,10 @@ neon-i8mm: &neon-i8mm neon-fp16: &neon-fp16 FnCall: [target_feature, ['enable = "neon,fp16"']] +# #[target_feature(enable = "neon,bf16")] +neon-bf16: &neon-bf16 + FnCall: [target_feature, ['enable = "neon,bf16"']] + # #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))] enable-fhm: &enable-fhm FnCall: [cfg_attr, [{ FnCall: [not, ['target_arch = "arm"']]}, { FnCall: [target_feature, ['enable = "fhm"']] }]] @@ -59,6 +63,18 @@ aarch64-crc-stable: &aarch64-crc-stable neon-unstable-f16: &neon-unstable-f16 FnCall: [unstable, ['feature = "stdarch_neon_f16"', 'issue = "136306"']] +# #[unstable(feature = "stdarch_neon_bf16", issue = "none")] +neon-unstable-bf16: &neon-unstable-bf16 + FnCall: [unstable, ['feature = "stdarch_neon_bf16"', 'issue = "none"']] + +# #[cfg(any(target_has_reliable_f16b, doc))] +cfg-reliable-bf16: &cfg-reliable-bf16 + FnCall: [cfg, [{ FnCall: [any, [target_has_reliable_f16b, doc]]}]] + +# #[doc(cfg(target_has_reliable_f16b))] +doc-cfg-reliable-bf16: &doc-cfg-reliable-bf16 + FnCall: [doc, [{ FnCall: [cfg, [target_has_reliable_f16b]]}]] + # #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")] neon-unstable-feat-lut: &neon-unstable-feat-lut FnCall: [unstable, ['feature = "stdarch_neon_feat_lut"', 'issue = "138050"']] @@ -14215,3 +14231,180 @@ intrinsics: compose: - FnCall: [static_assert_uimm_bits!, [IMM5, "1"]] - FnCall: ['simd_extract!', [a, 'IMM5 as u32'], [] , true] + +########## bf16 intrinsics + - name: "vreinterpret{neon_type[1].no}{neon_type[0].noq}" + doc: Vector reinterpret cast operation + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-unstable-bf16 + - *target-not-arm64ec + assert_instr: [nop] + safety: safe + types: + - [float64x1_t, bfloat16x4_t] + - [bfloat16x4_t, float64x1_t] + - [float64x2_t, bfloat16x8_t] + - [bfloat16x8_t, float64x2_t] + compose: + - FnCall: [transmute, [a]] + + - name: "vcopy{neon_type[0].lane_nox}" + doc: "Insert vector element from another vector element" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'LANE1 = {type[2]}', 'LANE2 = {type[3]}']]}]] + - FnCall: [rustc_legacy_const_generics, ['1', '3']] + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-unstable-bf16 + static_defs: ['const LANE1: i32, const LANE2: i32'] + safety: safe + types: + - [bfloat16x4_t, bfloat16x4_t, '3', '3', 'vcopy_lane_u16'] + - [bfloat16x8_t, bfloat16x4_t, '7', '3', 'vcopyq_lane_u16'] + compose: + - FnCall: [transmute, [FnCall: ['{type[4]}::', [FnCall: [transmute, [a]], FnCall: [transmute, [b]]]]]] + + - name: "vcopy{neon_type[0].laneq_nox}" + doc: "Insert vector element from another vector element" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'LANE1 = {type[2]}', 'LANE2 = {type[3]}']]}]] + - FnCall: [rustc_legacy_const_generics, ['1', '3']] + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-unstable-bf16 + static_defs: ['const LANE1: i32, const LANE2: i32'] + safety: safe + types: + - [bfloat16x4_t, bfloat16x8_t, '3', '7', 'vcopy_laneq_u16'] + - [bfloat16x8_t, bfloat16x8_t, '7', '7', 'vcopyq_laneq_u16'] + compose: + - FnCall: [transmute, [FnCall: ['{type[4]}::', [FnCall: [transmute, [a]], FnCall: [transmute, [b]]]]]] + + # MSVC's dumpbin prints raw opcodes for BF16 LUT instructions. + - name: "vluti2{neon_type[0].lane_nox}" + doc: "Lookup table read with 2-bit indices" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "bfloat16x8_t" + attr: + - FnCall: [target_feature, ['enable = "neon,lut,bf16"']] + - FnCall: [cfg_attr, [*msvc-disabled, {FnCall: [assert_instr, [luti2, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-unstable-bf16 + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [bfloat16x4_t, uint8x8_t, 'LANE >= 0 && LANE <= 3'] + - [bfloat16x8_t, uint8x8_t, 'LANE >= 0 && LANE <= 3'] + compose: + - FnCall: ['static_assert!', ['{type[2]}']] + - LLVMLink: + name: "vluti2{neon_type[0].lane_nox}" + arguments: + - 'a: {neon_type[0]}' + - 'b: {neon_type[1]}' + - 'n: i32' + links: + - link: "llvm.aarch64.neon.vluti2.lane.v8bf16.{neon_type[0]}" + arch: aarch64,arm64ec + - FnCall: ['_vluti2{neon_type[0].lane_nox}', [a, b, LANE]] + + - name: "vluti2{neon_type[0].laneq_nox}" + doc: "Lookup table read with 2-bit indices" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "bfloat16x8_t" + attr: + - FnCall: [target_feature, ['enable = "neon,lut,bf16"']] + - FnCall: [cfg_attr, [*msvc-disabled, {FnCall: [assert_instr, [luti2, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-unstable-bf16 + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [bfloat16x4_t, uint8x16_t, 'LANE >= 0 && LANE <= 7'] + - [bfloat16x8_t, uint8x16_t, 'LANE >= 0 && LANE <= 7'] + compose: + - FnCall: ['static_assert!', ['{type[2]}']] + - LLVMLink: + name: "vluti2{neon_type[0].laneq_nox}" + arguments: + - 'a: {neon_type[0]}' + - 'b: {neon_type[1]}' + - 'n: i32' + links: + - link: "llvm.aarch64.neon.vluti2.laneq.v8bf16.{neon_type[0]}" + arch: aarch64,arm64ec + - FnCall: ['_vluti2{neon_type[0].laneq_nox}', [a, b, LANE]] + + - name: "vluti4q_lane_bf16_x2" + doc: "Lookup table read with 4-bit indices" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "bfloat16x8_t" + attr: + - FnCall: [target_feature, ['enable = "neon,lut,bf16"']] + - FnCall: [cfg_attr, [*msvc-disabled, {FnCall: [assert_instr, [luti4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-unstable-bf16 + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [bfloat16x8x2_t, uint8x8_t, 'LANE >= 0 && LANE <= 1'] + compose: + - FnCall: ['static_assert!', ['{type[2]}']] + - LLVMLink: + name: "vluti4q_lane_bf16_x2" + arguments: + - 'a0: bfloat16x8_t' + - 'a1: bfloat16x8_t' + - 'b: {neon_type[1]}' + - 'n: i32' + links: + - link: "llvm.aarch64.neon.vluti4q.lane.x2.v8bf16" + arch: aarch64,arm64ec + - FnCall: ['_vluti4q_lane_bf16_x2', [a.0, a.1, b, LANE]] + + - name: "vluti4q_laneq_bf16_x2" + doc: "Lookup table read with 4-bit indices" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "bfloat16x8_t" + attr: + - FnCall: [target_feature, ['enable = "neon,lut,bf16"']] + - FnCall: [cfg_attr, [*msvc-disabled, {FnCall: [assert_instr, [luti4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-unstable-bf16 + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [bfloat16x8x2_t, uint8x16_t, 'LANE >= 0 && LANE <= 3'] + compose: + - FnCall: ['static_assert!', ['{type[2]}']] + - LLVMLink: + name: "vluti4q_laneq_bf16_x2" + arguments: + - 'a0: bfloat16x8_t' + - 'a1: bfloat16x8_t' + - 'b: {neon_type[1]}' + - 'n: i32' + links: + - link: "llvm.aarch64.neon.vluti4q.laneq.x2.v8bf16" + arch: aarch64,arm64ec + - FnCall: ['_vluti4q_laneq_bf16_x2', [a.0, a.1, b, LANE]] diff --git a/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml b/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml index 35c2a108c3..e80c4b2f8d 100644 --- a/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml +++ b/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml @@ -55,6 +55,10 @@ not-arm: ¬-arm neon-target-aarch64-arm64ec: &neon-target-aarch64-arm64ec FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]] +# MSVC's dumpbin prints raw opcodes for these BF16 instructions. +neon-target-aarch64-arm64ec-no-msvc: &neon-target-aarch64-arm64ec-no-msvc + FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}, {FnCall: [not, ['target_env = "msvc"']]}]] + neon-target-aarch64-le: &neon-target-aarch64-le FnCall: [all, [test, 'target_arch = "aarch64"', 'target_endian = "little"', {FnCall: [not, [windows]]}]] @@ -99,6 +103,10 @@ neon-i8mm: &neon-i8mm neon-fp16: &neon-fp16 FnCall: [target_feature, ['enable = "neon,fp16"']] +# #[target_feature(enable = "neon,bf16")] +neon-bf16: &neon-bf16 + FnCall: [target_feature, ['enable = "neon,bf16"']] + # #[cfg_attr[target_arch = "arm", target_feature(enable = "neon,fp16")] arm-fp16: &arm-fp16 FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [target_feature, ['enable = "fp16"']]}]] @@ -128,6 +136,18 @@ neon-unstable-f16: &neon-unstable-f16 all-neon-target-aarch64-arm64ec-little-endian: &all-neon-target-aarch64-arm64ec-little-endian FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}, 'target_endian = "little"']] +# #[cfg_attr(not(target_arch = "arm"), unstable(feature = "stdarch_neon_bf16", issue = "none"))] +neon-unstable-bf16: &neon-unstable-bf16 + FnCall: [cfg_attr, [*not-arm, {FnCall: [unstable, ['feature = "stdarch_neon_bf16"', 'issue = "none"']]}]] + +# #[cfg(any(target_has_reliable_f16b, doc))] +cfg-reliable-bf16: &cfg-reliable-bf16 + FnCall: [cfg, [{ FnCall: [any, [target_has_reliable_f16b, doc]]}]] + +# #[doc(cfg(target_has_reliable_f16b))] +doc-cfg-reliable-bf16: &doc-cfg-reliable-bf16 + FnCall: [doc, [{ FnCall: [cfg, [target_has_reliable_f16b]]}]] + intrinsics: - name: "vand{neon_type.no}" doc: Vector bitwise and @@ -15338,3 +15358,1349 @@ intrinsics: - ['vrev64q_f32', 'float32x4_t', 'vrev64.32', 'rev64', '[1, 0, 3, 2]'] compose: - FnCall: ['simd_shuffle!', [a, a, '{type[4]}']] + +##### bf16 intrinsics + - name: "vbfdot{neon_type[0].no}" + doc: "BFloat16 floating-point dot product (vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vdot.bf16"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfdot]]}]] + safety: safe + types: + - [float32x2_t, bfloat16x4_t, bfloat16x4_t] + - [float32x4_t, bfloat16x8_t, bfloat16x8_t] + compose: + - LLVMLink: + name: "bfdot.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.bfdot.{neon_type[0]}.{neon_type[1]}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.bfdot.{neon_type[0]}.{neon_type[1]}" + arch: arm + + - name: "vbfdot{neon_type[0].lane_nox}" + doc: "BFloat16 floating-point dot product (vector, by element)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vdot.bf16"', 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfdot, 'LANE = 0']]}]] + safety: safe + static_defs: ['const LANE: i32'] + types: + - [float32x2_t, bfloat16x4_t, bfloat16x4_t, 'vdup_lane_u32'] + - [float32x4_t, bfloat16x8_t, bfloat16x4_t, 'vdupq_lane_u32'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - Let: [pairs, "{neon_type[2]}", "c"] + - Let: [duped, '{neon_type[1]}', FnCall: [transmute, [FnCall: ['{type[3]}::', [FnCall: [transmute, [pairs]]]]]]] + - FnCall: ['vbfdot{neon_type[0].no}', [a, b, duped]] + + - name: "vbfdot{neon_type[0].laneq_nox}" + doc: "BFloat16 floating-point dot product (vector, by element)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vdot.bf16"', 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfdot, 'LANE = 0']]}]] + safety: safe + static_defs: ['const LANE: i32'] + types: + - [float32x4_t, bfloat16x8_t, bfloat16x8_t, 'vdupq_laneq_u32'] + - [float32x2_t, bfloat16x4_t, bfloat16x8_t, 'vdup_laneq_u32'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '2']] + - Let: [pairs, "{neon_type[2]}", "c"] + - Let: [duped, '{neon_type[1]}', FnCall: [transmute, [FnCall: ['{type[3]}::', [FnCall: [transmute, [pairs]]]]]]] + - FnCall: ['vbfdot{neon_type[0].no}', [a, b, duped]] + + - name: "vbfmlalbq_f32" + big_endian_inverse: true + doc: "BFloat16 floating-point widening multiply-add long (vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vfmab.bf16"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfmlalb]]}]] + safety: safe + types: + - [float32x4_t, bfloat16x8_t, bfloat16x8_t] + compose: + - LLVMLink: + name: "vbfmlalbq_f32" + links: + - link: "llvm.aarch64.neon.bfmlalb" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.bfmlalb" + arch: arm + +# "vbfmlalbq_lane_f32" ARM + - name: "vbfmlalbq_lane_f32" + doc: "BFloat16 floating-point widening multiply-add long (vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vfmab.bf16"', 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfmlalb, 'LANE = 0']]}]] + safety: safe + static_defs: ['const LANE: i32'] + types: + - [float32x4_t, bfloat16x8_t, bfloat16x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '2']] + - Let: [duped, bfloat16x8_t, FnCall: [transmute, [FnCall: ['vdupq_lane_u16::', [FnCall: [transmute, [c]]]]]]] + - FnCall: [vbfmlalbq_f32, [a, b, duped]] + + - name: "vbfmlalbq_laneq_f32" + doc: "BFloat16 floating-point widening multiply-add long (vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vfmab.bf16"', 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfmlalb, 'LANE = 0']]}]] + safety: safe + static_defs: ['const LANE: i32'] + types: + - [float32x4_t, bfloat16x8_t, bfloat16x8_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '3']] + - Let: [duped, bfloat16x8_t, FnCall: [transmute, [FnCall: ['vdupq_laneq_u16::', [FnCall: [transmute, [c]]]]]]] + - FnCall: [vbfmlalbq_f32, [a, b, duped]] + + - name: "vbfmlaltq_f32" + big_endian_inverse: true + doc: "BFloat16 floating-point widening multiply-add long (vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vfmat.bf16"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfmlalt]]}]] + safety: safe + types: + - [float32x4_t, bfloat16x8_t, bfloat16x8_t] + compose: + - LLVMLink: + name: "vbfmlalbq_f32" + links: + - link: "llvm.aarch64.neon.bfmlalt" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.bfmlalt" + arch: arm + +# "vbfmlalbq_lane_f32" ARM + - name: "vbfmlaltq_lane_f32" + doc: "BFloat16 floating-point widening multiply-add long (vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vfmat.bf16"', 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfmlalt, 'LANE = 0']]}]] + safety: safe + static_defs: ['const LANE: i32'] + types: + - [float32x4_t, bfloat16x8_t, bfloat16x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '2']] + - Let: [duped, bfloat16x8_t, FnCall: [transmute, [FnCall: ['vdupq_lane_u16::', [FnCall: [transmute, [c]]]]]]] + - FnCall: [vbfmlaltq_f32, [a, b, duped]] + + - name: "vbfmlaltq_laneq_f32" + doc: "BFloat16 floating-point widening multiply-add long (vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vfmat.bf16"', 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfmlalt, 'LANE = 0']]}]] + safety: safe + static_defs: ['const LANE: i32'] + types: + - [float32x4_t, bfloat16x8_t, bfloat16x8_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '3']] + - Let: [duped, bfloat16x8_t, FnCall: [transmute, [FnCall: ['vdupq_laneq_u16::', [FnCall: [transmute, [c]]]]]]] + - FnCall: [vbfmlaltq_f32, [a, b, duped]] + + - name: "vbfmmlaq_f32" + big_endian_inverse: true + doc: "BFloat16 floating-point matrix multiply-accumulate into 2x2 matrix" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmmla.bf16"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfmmla]]}]] + safety: safe + types: + - [float32x4_t, bfloat16x8_t, bfloat16x8_t] + compose: + - LLVMLink: + name: "bfmmla.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.bfmmla" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.bfmmla" + arch: arm + + - name: "vcombine_bf16" + doc: Join two smaller vectors into a single larger vector + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: safe + big_endian_inverse: true + types: + - [bfloat16x4_t, bfloat16x8_t] + compose: + - FnCall: [simd_shuffle!, [a, b, '[0, 1, 2, 3, 4, 5, 6, 7]']] + + - name: "vcreate_bf16" + doc: Insert vector element from another vector element + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: + unsafe: [neon] + types: + - ['u64', 'bfloat16x4_t'] + compose: + - FnCall: [transmute, [a]] + + - name: "vcvt_bf16_f32" + doc: "Floating-point convert from single-precision to BFloat16 format (vector)" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - *target-is-arm + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vcvt.bf16.f32"']]}]] + safety: safe + types: + - [float32x4_t, bfloat16x4_t] + compose: + - LLVMLink: + name: "vcvt_bf16_f32" + links: + - link: "llvm.arm.neon.vcvtfp2bf.v4bf16" + arch: arm + + - name: "vcvt_bf16_f32" + doc: "Floating-point convert from single-precision to BFloat16 format (vector)" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-unstable-bf16 + - *target-not-arm + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [bfcvtn]]}]] + safety: safe + types: + - [float32x4_t, bfloat16x4_t] + compose: + - FnCall: [vget_low_bf16, [FnCall: [vcvtq_low_bf16_f32, [a], [], true]], [], true] + + - name: "vcvt_f32_bf16" + doc: "Shift Left Long (by element size)" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vshll.i16"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [shll]]}]] + safety: safe + types: + - [bfloat16x4_t, float32x4_t] + compose: + - Let: [half, uint16x4_t, FnCall: [transmute, [a]]] + - FnCall: [vreinterpretq_f32_u32, [FnCall: ['vshll_n_u16::<16>', [half]]]] + + - name: "vcvtah_f32_bf16" + doc: "Shift Left (immediate)" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: safe + types: + - ['bfloat16_t', float32_t] + compose: + - FnCall: [f32::from_bits, ['(a.to_bits() as u32) << 16']] + + - name: "vcvth_bf16_f32" + doc: "Floating-point convert from single-precision to BFloat16 format (scalar)" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vcvtb.bf16.f32"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec-no-msvc, {FnCall: [assert_instr, [bfcvt]]}]] + safety: safe + types: + - [float32_t, bfloat16_t] + compose: + - LLVMLink: + name: "vcvth_bf16_f32" + links: + - link: "llvm.arm.neon.vcvtbfp2bf" + arch: arm + - link: "llvm.aarch64.neon.bfcvt" + arch: aarch64,arm64ec + + - name: "vcvtq_low_bf16_f32" + big_endian_inverse: true + doc: "Floating-point convert from single-precision to BFloat16 format (vector)" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - *target-is-arm + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vcvt.bf16.f32"']]}]] + safety: + unsafe: [neon] + types: + - [float32x4_t, bfloat16x8_t] + compose: + - Let: [a, int32x4_t, FnCall: [transmute, [a]]] + - Let: [a, int8x16_t, FnCall: [transmute, [a]]] + - Let: [a, float32x4_t, FnCall: [transmute, [a]]] + - Let: [vec, bfloat16x4_t, FnCall: [vcvt_bf16_f32, [a]]] + - Let: [z, bfloat16x4_t, 'bfloat16x4_t([bfloat16_t::from_bits(0x0), bfloat16_t::from_bits(0x0), bfloat16_t::from_bits(0x0), bfloat16_t::from_bits(0x0)])'] + - FnCall: [simd_shuffle!, [z, vec, '[0, 1, 2, 3, 4, 5, 6, 7]']] + + - name: "vcvtq_low_bf16_f32" + big_endian_inverse: true + doc: "Floating-point convert from single-precision to BFloat16 format (vector)" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - *target-not-arm + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [bfcvtn]]}]] + safety: + unsafe: [neon] + types: + - [float32x4_t, bfloat16x8_t] + compose: + - LLVMLink: + name: "vcvtq_low_bf16_f32" + links: + - link: "llvm.aarch64.neon.bfcvtn" + arch: aarch64,arm64ec + + - name: "vcvtq_high_f32_bf16" + doc: "Shift Left Long (by element size)" + arguments: ["a: {type[1]}"] + return_type: "{type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vshll.i16"']]}]] + - FnCall: [cfg_attr, [*all-neon-target-aarch64-arm64ec-little-endian, {FnCall: [assert_instr, [shll2]]}]] + # Reversing big-endian lanes lets LLVM use the low-half instruction. + - FnCall: [cfg_attr, [*neon-target-aarch64-be, {FnCall: [assert_instr, [shll]]}]] + safety: safe + types: + - [float32x4_t, bfloat16x8_t] + compose: + - Let: [half, uint16x4_t, FnCall: [vget_high_u16, [FnCall: [transmute, [a]]]]] + - FnCall: [vreinterpretq_f32_u32, [FnCall: ['vshll_n_u16::<16>', [half]]]] + + - name: "vcvtq_high_bf16_f32" + big_endian_inverse: true + doc: "Floating-point convert from single-precision to BFloat16 format (vector)" + arguments: ["_inactive: {neon_type[0]}", "a: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - *target-not-arm + - FnCall: [cfg_attr, [*all-neon-target-aarch64-arm64ec-little-endian, {FnCall: [assert_instr, [bfcvtn2]]}]] + # Reversing big-endian lanes lets LLVM use the low-half instruction. + - FnCall: [cfg_attr, [*neon-target-aarch64-be, {FnCall: [assert_instr, [bfcvtn]]}]] + safety: + unsafe: [neon] + types: + - [bfloat16x8_t, float32x4_t] + compose: + - LLVMLink: + name: "vcvtq_high_bf16_f32" + links: + - link: "llvm.aarch64.neon.bfcvtn2" + arch: aarch64,arm64ec + + # T + - name: "vcvtq_high_bf16_f32" + doc: "Floating-point convert from single-precision to BFloat16 format (vector)" + arguments: ["_inactive: {neon_type[0]}", "a: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - *target-is-arm + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vcvt.bf16.f32"']]}]] + safety: + unsafe: [neon] + types: + - [bfloat16x8_t, float32x4_t] + compose: + - Let: [b, bfloat16x4_t, FnCall: [simd_shuffle!, [_inactive, _inactive, '[0, 1, 2, 3]']]] + - Let: [a, int32x4_t, FnCall: [transmute, [a]]] + - Let: [a, int16x8_t, FnCall: [transmute, [a]]] + - Let: [a, float32x4_t, FnCall: [transmute, [a]]] + - Let: [a, bfloat16x4_t, FnCall: [vcvt_bf16_f32, [a]]] + - Let: [vec, bfloat16x4_t, FnCall: [simd_shuffle!, [a, a, '[0, 1, 2, 3]']]] + - FnCall: [simd_shuffle!, [vec, b, '[0, 1, 2, 3, 4, 5, 6, 7]']] + + - name: "vcvtq_low_f32_bf16" + doc: "Shift Left Long (by element size)" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vshll.i16"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [shll]]}]] + safety: safe + types: + - [bfloat16x8_t, float32x4_t] + compose: + - Let: [half, uint16x4_t, FnCall: [vget_low_u16, [FnCall: [transmute, [a]]]]] + - FnCall: [vreinterpretq_f32_u32, [FnCall: ['vshll_n_u16::<16>', [half]]]] + + - name: "vdup_lane_bf16" + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: bfloat16x4_t"] + return_type: "{neon_type[0]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + safety: + unsafe: [neon] + static_defs: ['const LANE: i32'] + types: + - [bfloat16x4_t, bfloat16x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, 2]] + - Let: [c, uint16x4_t, FnCall: [vreinterpret_u16_bf16, [a]]] + - Let: [duped, uint16x4_t, FnCall: ['vdup_lane_u16::', [c]]] + - FnCall: [vreinterpret_bf16_u16, [duped]] + + - name: "vdupq_lane_bf16" + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + safety: + unsafe: [neon] + static_defs: ['const LANE: i32'] + types: + - [bfloat16x4_t, bfloat16x8_t] + compose: + # cast to u16s, duplicate the lane using a u16 intrinsic and then cast back + - FnCall: [static_assert_uimm_bits!, [LANE, 2]] + - Let: [c, uint16x4_t, FnCall: [vreinterpret_u16_bf16, [a]]] + - Let: [duped, uint16x8_t, FnCall: ['vdupq_lane_u16::', [c]]] + - FnCall: [vreinterpretq_bf16_u16, [duped]] + + - name: "vdup_n_bf16" + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: + unsafe: [neon] + types: + - [bfloat16_t, bfloat16x4_t] + compose: + - FnCall: [vreinterpret_bf16_u16, [FnCall: [vdup_n_u16, [MethodCall: [a, to_bits, []]]]]] + + - name: "vdupq_n_bf16" + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: + unsafe: [neon] + types: + - [bfloat16_t, bfloat16x8_t] + compose: + - FnCall: [vreinterpretq_bf16_u16, [FnCall: [vdupq_n_u16, [MethodCall: [a, to_bits, []]]]]] + + - name: "vget_lane_bf16" + big_endian_inverse: true + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + safety: + unsafe: [neon] + static_defs: ['const LANE: i32'] + types: + - [bfloat16x4_t, bfloat16_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '2']] + - FnCall: [simd_extract!, [a, 'LANE as u32']] + + - name: "vgetq_lane_bf16" + big_endian_inverse: true + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + safety: + unsafe: [neon] + static_defs: ['const LANE: i32'] + types: + - [bfloat16x8_t, bfloat16_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '3']] + - FnCall: [simd_extract!, [a, 'LANE as u32']] + + - name: "vduph_lane_bf16" + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - [bfloat16x4_t, bfloat16_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, 2]] + - FnCall: ['vget_lane_bf16::', [a]] + + - name: "vduph_laneq_bf16" + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - [bfloat16x8_t, bfloat16_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '3']] + - FnCall: ['vgetq_lane_bf16::', [a]] + + - name: "vdup{neon_type[1].laneq_nox}" + big_endian_inverse: true + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - [bfloat16x8_t, bfloat16x4_t] + - [bfloat16x8_t, bfloat16x8_t] + compose: + - FnCall: ["static_assert_uimm_bits!", [LANE, '3']] + - FnCall: ["simd_shuffle!", [a, a, '[LANE as u32; {neon_type[1].lane}]']] + + - name: "vget_high_bf16" + big_endian_inverse: true + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: + unsafe: [neon] + types: + - [bfloat16x8_t, bfloat16x4_t] + compose: + - FnCall: [simd_shuffle!, [a, a, "[4, 5, 6, 7]"]] + + - name: "vget_low_bf16" + doc: "Duplicate vector element to vector or scalar" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: + unsafe: [neon] + types: + - [bfloat16x8_t, bfloat16x4_t] + compose: + - FnCall: [transmute, [FnCall: [vget_low_u64, [FnCall: [transmute, [a]]]]]] + + - name: "vld1{neon_type[1].no}" + doc: "Load multiple single-element structures to one, two, three, or four registers" + arguments: ["ptr: *const {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: + unsafe: [neon] + types: + - [bfloat16_t, bfloat16x4_t] + - [bfloat16_t, bfloat16x8_t] + compose: + - FnCall: ['crate::ptr::read_unaligned', [MethodCall: [ptr, cast, []]]] + + - name: "{type[0]}" + doc: "Load multiple single-element structures to one, two, three, or four registers" + arguments: ["ptr: *const {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *target-is-arm + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vld1.8"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] + safety: + unsafe: [neon] + types: + - ['vld1_bf16_x2', bfloat16_t, bfloat16x4x2_t, "ld1x2.v4bf16.p0", 'vld1_u16_x2'] + - ['vld1q_bf16_x2', bfloat16_t, bfloat16x8x2_t, "ld1x2.v8bf16.p0", 'vld1q_u16_x2'] + - ['vld1_bf16_x3', bfloat16_t, bfloat16x4x3_t, "ld1x3.v4bf16.p0", 'vld1_u16_x3'] + - ['vld1q_bf16_x3', bfloat16_t, bfloat16x8x3_t, "ld1x3.v8bf16.p0", 'vld1q_u16_x3'] + - ['vld1_bf16_x4', bfloat16_t, bfloat16x4x4_t, "ld1x4.v4bf16.p0", 'vld1_u16_x4'] + - ['vld1q_bf16_x4', bfloat16_t, bfloat16x8x4_t, "ld1x4.v8bf16.p0", 'vld1q_u16_x4'] + compose: + - FnCall: [transmute, [FnCall: ['{type[4]}', [MethodCall: [ptr, cast, []]]]]] + + - name: "{type[0]}" + doc: "Load multiple single-element structures to one, two, three, or four registers" + arguments: ["ptr: *const {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *target-not-arm + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vld1.8"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] + safety: + unsafe: [neon] + types: + - ['vld1_bf16_x2', bfloat16_t, bfloat16x4x2_t, "ld1x2.v4bf16.p0", 'vld1_u16_x2'] + - ['vld1q_bf16_x2', bfloat16_t, bfloat16x8x2_t, "ld1x2.v8bf16.p0", 'vld1q_u16_x2'] + - ['vld1_bf16_x3', bfloat16_t, bfloat16x4x3_t, "ld1x3.v4bf16.p0", 'vld1_u16_x3'] + - ['vld1q_bf16_x3', bfloat16_t, bfloat16x8x3_t, "ld1x3.v8bf16.p0", 'vld1q_u16_x3'] + - ['vld1_bf16_x4', bfloat16_t, bfloat16x4x4_t, "ld1x4.v4bf16.p0", 'vld1_u16_x4'] + - ['vld1q_bf16_x4', bfloat16_t, bfloat16x8x4_t, "ld1x4.v8bf16.p0", 'vld1q_u16_x4'] + compose: + - LLVMLink: + name: "{type[0]}" + links: + - link: "llvm.aarch64.neon.{type[3]}" + arch: aarch64,arm64ec + + - name: "{type[0]}" + doc: "Load one single-element structure and Replicate to all lanes (of one register)" + arguments: ["ptr: *const {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: + unsafe: [neon] + types: + - ['vld1_dup_bf16', bfloat16_t, bfloat16x4_t, 'f16bx4::splat', 'vld1_dup_u16'] + - ['vld1q_dup_bf16', bfloat16_t, bfloat16x8_t, 'f16bx8::splat', 'vld1q_dup_u16'] + compose: + - FnCall: [transmute, [FnCall: ['{type[4]}', [MethodCall: [ptr, cast, []]]]]] + + - name: "{type[0]}" + doc: "Load one single-element structure to one lane of one register" + arguments: ["ptr: *const {type[1]}", "src: {neon_type[2]}"] + return_type: "{neon_type[2]}" + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - ['vld1_lane_bf16', bfloat16_t, bfloat16x4_t, '2', 'vld1_lane_u16'] + - ['vld1q_lane_bf16', bfloat16_t, bfloat16x8_t, '3', 'vld1q_lane_u16'] + compose: + - FnCall: [transmute, [FnCall: ['{type[4]}::', [MethodCall: [ptr, cast, []], FnCall: [transmute, [src]]]]]] + + - name: "{type[0]}" + doc: "Load multiple 2-element structures to two registers" + arguments: ["ptr: *const {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *target-is-arm + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"v{type[4]}.16"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['{type[4]}']]}]] + safety: + unsafe: [neon] + types: + - ['vld2_bf16', bfloat16_t, bfloat16x4x2_t, "ld2.v4bf16.p0", 'ld2', 'vld2_u16'] + - ['vld2q_bf16', bfloat16_t, bfloat16x8x2_t, "ld2.v8bf16.p0", 'ld2', 'vld2q_u16'] + - ['vld3_bf16', bfloat16_t, bfloat16x4x3_t, "ld3.v4bf16.p0", 'ld3', 'vld3_u16'] + - ['vld3q_bf16', bfloat16_t, bfloat16x8x3_t, "ld3.v8bf16.p0", 'ld3', 'vld3q_u16'] + - ['vld4_bf16', bfloat16_t, bfloat16x4x4_t, "ld4.v4bf16.p0", 'ld4', 'vld4_u16'] + - ['vld4q_bf16', bfloat16_t, bfloat16x8x4_t, "ld4.v8bf16.p0", 'ld4', 'vld4q_u16'] + compose: + - FnCall: [transmute, [FnCall: ['{type[5]}', [MethodCall: [ptr, cast, []]]]]] + + - name: "{type[0]}" + doc: "Load multiple 2-element structures to two registers" + arguments: ["ptr: *const {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *target-not-arm + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"v{type[4]}.16"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['{type[4]}']]}]] + safety: + unsafe: [neon] + types: + - ['vld2_bf16', bfloat16_t, bfloat16x4x2_t, "ld2.v4bf16.p0", 'ld2', 'vld2_u16'] + - ['vld2q_bf16', bfloat16_t, bfloat16x8x2_t, "ld2.v8bf16.p0", 'ld2', 'vld2q_u16'] + - ['vld3_bf16', bfloat16_t, bfloat16x4x3_t, "ld3.v4bf16.p0", 'ld3', 'vld3_u16'] + - ['vld3q_bf16', bfloat16_t, bfloat16x8x3_t, "ld3.v8bf16.p0", 'ld3', 'vld3q_u16'] + - ['vld4_bf16', bfloat16_t, bfloat16x4x4_t, "ld4.v4bf16.p0", 'ld4', 'vld4_u16'] + - ['vld4q_bf16', bfloat16_t, bfloat16x8x4_t, "ld4.v8bf16.p0", 'ld4', 'vld4q_u16'] + compose: + - LLVMLink: + name: "{type[0]}" + links: + - link: "llvm.aarch64.neon.{type[3]}" + arch: aarch64,arm64ec + + - name: "{type[0]}" + doc: "Load single 2-element structure and Replicate to all lanes of two registers" + arguments: ["ptr: *const {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"v{type[4]}.16"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['{type[4]}']]}]] + safety: + unsafe: [neon] + types: + - ['vld2_dup_bf16', bfloat16_t, bfloat16x4x2_t, "ld2r.v4bf16.p0", 'ld2', 'vld2_dup_u16'] + - ['vld2q_dup_bf16', bfloat16_t, bfloat16x8x2_t, "ld2dup.v8bf16.p0", 'ld2', 'vld2q_dup_u16'] + - ['vld3_dup_bf16', bfloat16_t, bfloat16x4x3_t, "ld3r.v4bf16.p0", 'ld3', 'vld3_dup_u16'] + - ['vld3q_dup_bf16', bfloat16_t, bfloat16x8x3_t, "ld3dup.v8bf16.p0", 'ld3', 'vld3q_dup_u16'] + - ['vld4_dup_bf16', bfloat16_t, bfloat16x4x4_t, "ld4r.v4bf16.p0", 'ld4', 'vld4_dup_u16'] + - ['vld4q_dup_bf16', bfloat16_t, bfloat16x8x4_t, "ld4dup.v8bf16.p0", 'ld4', 'vld4q_dup_u16'] + compose: + - FnCall: [transmute, [FnCall: ['{type[5]}', [MethodCall: [ptr, cast, []]]]]] + + - name: "{type[0]}" + doc: "Load single 2-element structure and Replicate to all lanes of two registers" + arguments: ["ptr: *const {type[1]}", "src: {neon_type[2]}"] + return_type: "{neon_type[2]}" + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"v{type[4]}.16"', 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['{type[4]}', 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - ['vld2_lane_bf16', bfloat16_t, bfloat16x4x2_t, "ld2lane.v4bf16.p0", 'ld2', '2', 'vld2_lane_u16'] + - ['vld2q_lane_bf16', bfloat16_t, bfloat16x8x2_t, "ld2lane.v8bf16.p0", 'ld2', '3', 'vld2q_lane_u16'] + compose: + - FnCall: [transmute, [FnCall: ['{type[6]}::', [MethodCall: [ptr, cast, []], FnCall: [transmute, [src]]]]]] + + - name: "{type[0]}" + doc: "Load single 3-element structure to one lane of three registers" + arguments: ["ptr: *const {type[1]}", "src: {neon_type[2]}"] + return_type: "{neon_type[2]}" + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"v{type[4]}.16"', 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['{type[4]}', 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - ['vld3_lane_bf16', bfloat16_t, bfloat16x4x3_t, "ld3lane.v4bf16.p0", 'ld3', '2', 'vld3_lane_u16'] + - ['vld3q_lane_bf16', bfloat16_t, bfloat16x8x3_t, "ld3lane.v8bf16.p0", 'ld3', '3', 'vld3q_lane_u16'] + compose: + - FnCall: [transmute, [FnCall: ['{type[6]}::', [MethodCall: [ptr, cast, []], FnCall: [transmute, [src]]]]]] + + - name: "{type[0]}" + doc: "Load single 4-element structure and Replicate to all lanes of four registers" + arguments: ["ptr: *const {type[1]}", "src: {neon_type[2]}"] + return_type: "{neon_type[2]}" + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"v{type[4]}.16"', 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['{type[4]}', 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - ['vld4_lane_bf16', bfloat16_t, bfloat16x4x4_t, "ld4lane.v4bf16.p0", 'ld4', '2', 'vld4_lane_u16'] + - ['vld4q_lane_bf16', bfloat16_t, bfloat16x8x4_t, "ld4lane.v8bf16.p0", 'ld4', '3', 'vld4q_lane_u16'] + compose: + - FnCall: [transmute, [FnCall: ['{type[6]}::', [MethodCall: [ptr, cast, []], FnCall: [transmute, [src]]]]]] + + - name: "{type[0]}" + doc: "Insert vector element from another vector element" + arguments: ["a: {type[1]}", "v: {neon_type[2]}"] + return_type: "{neon_type[2]}" + big_endian_inverse: true + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['nop', 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['nop', 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - ['vset_lane_bf16', bfloat16_t, bfloat16x4_t, '2'] + - ['vsetq_lane_bf16', bfloat16_t, bfloat16x8_t, '3'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: ['simd_insert!', [v, 'LANE as u32', a]] + + - name: "{type[0]}" + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["ptr: *mut {type[1]}", "val: {neon_type[2]}"] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vst1.8"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [str]]}]] + safety: + unsafe: [neon] + types: + - ['vst1_bf16', bfloat16_t, bfloat16x4_t, '2', 'vst1_u16'] + - ['vst1q_bf16', bfloat16_t, bfloat16x8_t, '3', 'vst1q_u16'] + compose: + - FnCall: ['crate::ptr::write_unaligned', [MethodCall: [ptr, cast, []], val]] + + - name: "{type[0]}" + doc: "Load single 2-element structure and Replicate to all lanes of two registers" + arguments: ["ptr: *mut {type[1]}", "val: {neon_type[2]}"] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"v{type[4]}.16"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['{type[4]}']]}]] + safety: + unsafe: [neon] + types: + - ['vst1_bf16_x2', bfloat16_t, bfloat16x4x2_t, "st1x2.p0.v4bf16", 'st1', 'vst1_u16_x2'] + - ['vst1q_bf16_x2', bfloat16_t, bfloat16x8x2_t, "st1x2.p0.v8bf16", 'st1', 'vst1q_u16_x2'] + - ['vst1_bf16_x3', bfloat16_t, bfloat16x4x3_t, "st1x3.p0.v4bf16", 'st1', 'vst1_u16_x3'] + - ['vst1q_bf16_x3', bfloat16_t, bfloat16x8x3_t, "st1x3.p0.v8bf16", 'st1', 'vst1q_u16_x3'] + - ['vst1_bf16_x4', bfloat16_t, bfloat16x4x4_t, "st1x4.p0.v4bf16", 'st1', 'vst1_u16_x4'] + - ['vst1q_bf16_x4', bfloat16_t, bfloat16x8x4_t, "st1x4.p0.v8bf16", 'st1', 'vst1q_u16_x4'] + compose: + - FnCall: ['{type[5]}', [MethodCall: [ptr, cast, []], FnCall: [transmute, [val]]]] + + - name: "{type[0]}" + doc: "Store a single-element structure from one lane of one register" + arguments: ["a: *mut {type[1]}", "v: {neon_type[2]}"] + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['nop', 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['nop', 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - ['vst1_lane_bf16', bfloat16_t, bfloat16x4_t, '2', 'vst1_lane_u16'] + - ['vst1q_lane_bf16', bfloat16_t, bfloat16x8_t, '3', 'vst1q_lane_u16'] + compose: + - FnCall: ['{type[4]}::', [MethodCall: [a, cast, []], FnCall: [transmute, [v]]]] + + - name: "{type[0]}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[1]}", "b: {neon_type[2]}"] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['nop', 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['nop', 'LANE = 0']]}]] + safety: + unsafe: [neon] + types: + - ['vst2_bf16', bfloat16_t, bfloat16x4x2_t, '4', 'vst2_u16'] + - ['vst2q_bf16', bfloat16_t, bfloat16x8x2_t, '8', 'vst2q_u16'] + compose: + - FnCall: ['{type[4]}', [MethodCall: [a, cast, []], FnCall: [transmute, [b]]]] + + - name: "{type[0]}" + doc: "Store single 2-element structure from one lane of two registers" + arguments: ["a: *mut {type[1]}", "v: {neon_type[2]}"] + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vst2.16"', 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['st2', 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - ['vst2_lane_bf16', bfloat16_t, bfloat16x4x2_t, 'st2lane.v4bf16', '2', 'vst2_lane_u16'] + - ['vst2q_lane_bf16', bfloat16_t, bfloat16x8x2_t, 'st2lane.v8bf16', '3', 'vst2q_lane_u16'] + compose: + - FnCall: ['{type[5]}::', [MethodCall: [a, cast, []], FnCall: [transmute, [v]]]] + + - name: "{type[0]}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[1]}", "b: {neon_type[2]}"] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['nop', 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['nop', 'LANE = 0']]}]] + safety: + unsafe: [neon] + types: + - ['vst3_bf16', bfloat16_t, bfloat16x4x3_t, '4', 'vst3_u16'] + - ['vst3q_bf16', bfloat16_t, bfloat16x8x3_t, '8', 'vst3q_u16'] + compose: + - FnCall: ['{type[4]}', [MethodCall: [a, cast, []], FnCall: [transmute, [b]]]] + + - name: "{type[0]}" + doc: "Store single 3-element structure from one lane of three registers" + arguments: ["a: *mut {type[1]}", "v: {neon_type[2]}"] + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vst3.16"', 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['st3', 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - ['vst3_lane_bf16', bfloat16_t, bfloat16x4x3_t, 'st3lane.v4bf16', '2', 'vst3_lane_u16'] + - ['vst3q_lane_bf16', bfloat16_t, bfloat16x8x3_t, 'st3lane.v8bf16', '3', 'vst3q_lane_u16'] + compose: + - FnCall: ['{type[5]}::', [MethodCall: [a, cast, []], FnCall: [transmute, [v]]]] + + - name: "{type[0]}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[1]}", "b: {neon_type[2]}"] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['nop', 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['nop', 'LANE = 0']]}]] + safety: + unsafe: [neon] + types: + - ['vst4_bf16', bfloat16_t, bfloat16x4x4_t, '4', 'vst4_u16'] + - ['vst4q_bf16', bfloat16_t, bfloat16x8x4_t, '8', 'vst4q_u16'] + compose: + - FnCall: ['{type[4]}', [MethodCall: [a, cast, []], FnCall: [transmute, [b]]]] + + - name: "{type[0]}" + doc: "Store single 4-element structure from one lane of four registerss" + arguments: ["a: *mut {type[1]}", "v: {neon_type[2]}"] + static_defs: ['const LANE: i32'] + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vst4.16"', 'LANE = 2']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['st4', 'LANE = 2']]}]] + safety: + unsafe: [neon] + types: + - ['vst4_lane_bf16', bfloat16_t, bfloat16x4x4_t, 'st4lane.v4bf16', '2', 'vst4_lane_u16'] + - ['vst4q_lane_bf16', bfloat16_t, bfloat16x8x4_t, 'st4lane.v8bf16', '3', 'vst4q_lane_u16'] + compose: + - FnCall: ['{type[5]}::', [MethodCall: [a, cast, []], FnCall: [transmute, [v]]]] + + - name: "vreinterpret{type[0]}" + doc: "Vector reinterpret cast operation" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: + unsafe: [neon] + types: + - ["_bf16_f16", "float16x4_t", "bfloat16x4_t"] + - ["_bf16_f32", "float32x2_t", "bfloat16x4_t"] + - ["_bf16_p16", "poly16x4_t", "bfloat16x4_t"] + - ["_bf16_p64", "poly64x1_t", "bfloat16x4_t"] + - ["_bf16_p8", "poly8x8_t", "bfloat16x4_t"] + - ["_bf16_s16", "int16x4_t", "bfloat16x4_t"] + - ["_bf16_s32", "int32x2_t", "bfloat16x4_t"] + - ["_bf16_s64", "int64x1_t", "bfloat16x4_t"] + - ["_bf16_s8", "int8x8_t", "bfloat16x4_t"] + - ["_bf16_u16", "uint16x4_t", "bfloat16x4_t"] + - ["_bf16_u32", "uint32x2_t", "bfloat16x4_t"] + - ["_bf16_u64", "uint64x1_t", "bfloat16x4_t"] + - ["_bf16_u8", "uint8x8_t", "bfloat16x4_t"] + - ["_f16_bf16", "bfloat16x4_t", "float16x4_t"] + - ["_f32_bf16", "bfloat16x4_t", "float32x2_t"] + - ["_p8_bf16", "bfloat16x4_t", "poly8x8_t"] + - ["_p16_bf16", "bfloat16x4_t", "poly16x4_t"] + - ["_p64_bf16", "bfloat16x4_t", "poly64x1_t"] + - ["_s8_bf16", "bfloat16x4_t", "int8x8_t"] + - ["_s16_bf16", "bfloat16x4_t", "int16x4_t"] + - ["_s32_bf16", "bfloat16x4_t", "int32x2_t"] + - ["_s64_bf16", "bfloat16x4_t", "int64x1_t"] + - ["_u8_bf16", "bfloat16x4_t", "uint8x8_t"] + - ["_u16_bf16", "bfloat16x4_t", "uint16x4_t"] + - ["_u32_bf16", "bfloat16x4_t", "uint32x2_t"] + - ["_u64_bf16", "bfloat16x4_t", "uint64x1_t"] + compose: + - FnCall: [transmute, [a]] + + - name: "vreinterpretq{type[0]}" + doc: "Vector reinterpret cast operation" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - *cfg-reliable-bf16 + - *doc-cfg-reliable-bf16 + - *neon-bf16 + - *neon-v8 + - *neon-unstable-bf16 + - *neon-cfg-arm-unstable + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + safety: + unsafe: [neon] + types: + - ["_bf16_f16", "float16x8_t", "bfloat16x8_t"] + - ["_bf16_f32", "float32x4_t", "bfloat16x8_t"] + - ["_bf16_p128", "p128", "bfloat16x8_t"] + - ["_bf16_p16", "poly16x8_t", "bfloat16x8_t"] + - ["_bf16_p64", "poly64x2_t", "bfloat16x8_t"] + - ["_bf16_p8", "poly8x16_t", "bfloat16x8_t"] + - ["_bf16_s16", "int16x8_t", "bfloat16x8_t"] + - ["_bf16_s32", "int32x4_t", "bfloat16x8_t"] + - ["_bf16_s64", "int64x2_t", "bfloat16x8_t"] + - ["_bf16_s8", "int8x16_t", "bfloat16x8_t"] + - ["_bf16_u16", "uint16x8_t", "bfloat16x8_t"] + - ["_bf16_u32", "uint32x4_t", "bfloat16x8_t"] + - ["_bf16_u64", "uint64x2_t", "bfloat16x8_t"] + - ["_bf16_u8", "uint8x16_t", "bfloat16x8_t"] + - ["_f16_bf16", "bfloat16x8_t", "float16x8_t"] + - ["_f32_bf16", "bfloat16x8_t", "float32x4_t"] + - ["_p128_bf16", "bfloat16x8_t", "p128"] + - ["_p16_bf16", "bfloat16x8_t", "poly16x8_t"] + - ["_p64_bf16", "bfloat16x8_t", "poly64x2_t"] + - ["_p8_bf16", "bfloat16x8_t", "poly8x16_t"] + - ["_s16_bf16", "bfloat16x8_t", "int16x8_t"] + - ["_s32_bf16", "bfloat16x8_t", "int32x4_t"] + - ["_s64_bf16", "bfloat16x8_t", "int64x2_t"] + - ["_s8_bf16", "bfloat16x8_t", "int8x16_t"] + - ["_u16_bf16", "bfloat16x8_t", "uint16x8_t"] + - ["_u32_bf16", "bfloat16x8_t", "uint32x4_t"] + - ["_u64_bf16", "bfloat16x8_t", "uint64x2_t"] + - ["_u8_bf16", "bfloat16x8_t", "uint8x16_t"] + compose: + - FnCall: [transmute, [a]] diff --git a/intrinsics_data/arm_intrinsics.json b/intrinsics_data/arm_intrinsics.json index 7f749fe4d8..171b042bc3 100644 --- a/intrinsics_data/arm_intrinsics.json +++ b/intrinsics_data/arm_intrinsics.json @@ -51174,15 +51174,15 @@ "svfloat16x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svfloat16_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svfloat16_t" - }, "Architectures": [ "A64" ] @@ -51194,15 +51194,15 @@ "svfloat32x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svfloat32_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svfloat32_t" - }, "Architectures": [ "A64" ] @@ -51214,15 +51214,15 @@ "svfloat64x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svfloat64_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svfloat64_t" - }, "Architectures": [ "A64" ] @@ -51234,15 +51234,15 @@ "svint16x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint16_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svint16_t" - }, "Architectures": [ "A64" ] @@ -51254,15 +51254,15 @@ "svint32x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint32_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svint32_t" - }, "Architectures": [ "A64" ] @@ -51274,15 +51274,15 @@ "svint64x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint64_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svint64_t" - }, "Architectures": [ "A64" ] @@ -51294,15 +51294,15 @@ "svint8x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint8_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svint8_t" - }, "Architectures": [ "A64" ] @@ -51314,15 +51314,15 @@ "svuint16x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint16_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svuint16_t" - }, "Architectures": [ "A64" ] @@ -51334,15 +51334,15 @@ "svuint32x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint32_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svuint32_t" - }, "Architectures": [ "A64" ] @@ -51354,15 +51354,15 @@ "svuint64x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint64_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svuint64_t" - }, "Architectures": [ "A64" ] @@ -51374,15 +51374,15 @@ "svuint8x2_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint8_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svuint8_t" - }, "Architectures": [ "A64" ] @@ -51394,15 +51394,15 @@ "svfloat16x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svfloat16_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svfloat16_t" - }, "Architectures": [ "A64" ] @@ -51414,15 +51414,15 @@ "svfloat32x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svfloat32_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svfloat32_t" - }, "Architectures": [ "A64" ] @@ -51434,15 +51434,15 @@ "svfloat64x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svfloat64_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svfloat64_t" - }, "Architectures": [ "A64" ] @@ -51454,15 +51454,15 @@ "svint16x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint16_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svint16_t" - }, "Architectures": [ "A64" ] @@ -51474,15 +51474,15 @@ "svint32x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint32_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svint32_t" - }, "Architectures": [ "A64" ] @@ -51494,15 +51494,15 @@ "svint64x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint64_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svint64_t" - }, "Architectures": [ "A64" ] @@ -51514,15 +51514,15 @@ "svint8x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint8_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svint8_t" - }, "Architectures": [ "A64" ] @@ -51534,15 +51534,15 @@ "svuint16x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint16_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svuint16_t" - }, "Architectures": [ "A64" ] @@ -51554,15 +51554,15 @@ "svuint32x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint32_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svuint32_t" - }, "Architectures": [ "A64" ] @@ -51574,15 +51574,15 @@ "svuint64x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint64_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svuint64_t" - }, "Architectures": [ "A64" ] @@ -51594,15 +51594,15 @@ "svuint8x3_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint8_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svuint8_t" - }, "Architectures": [ "A64" ] @@ -51614,15 +51614,15 @@ "svfloat16x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svfloat16_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svfloat16_t" - }, "Architectures": [ "A64" ] @@ -51634,15 +51634,15 @@ "svfloat32x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svfloat32_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svfloat32_t" - }, "Architectures": [ "A64" ] @@ -51654,15 +51654,15 @@ "svfloat64x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svfloat64_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svfloat64_t" - }, "Architectures": [ "A64" ] @@ -51674,15 +51674,15 @@ "svint16x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint16_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svint16_t" - }, "Architectures": [ "A64" ] @@ -51694,15 +51694,15 @@ "svint32x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint32_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svint32_t" - }, "Architectures": [ "A64" ] @@ -51714,15 +51714,15 @@ "svint64x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint64_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svint64_t" - }, "Architectures": [ "A64" ] @@ -51734,15 +51734,15 @@ "svint8x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svint8_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svint8_t" - }, "Architectures": [ "A64" ] @@ -51754,15 +51754,15 @@ "svuint16x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint16_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svuint16_t" - }, "Architectures": [ "A64" ] @@ -51774,15 +51774,15 @@ "svuint32x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint32_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svuint32_t" - }, "Architectures": [ "A64" ] @@ -51794,15 +51794,15 @@ "svuint64x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint64_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svuint64_t" - }, "Architectures": [ "A64" ] @@ -51814,15 +51814,15 @@ "svuint8x4_t tuple", "uint64_t imm_index" ], + "return_type": { + "value": "svuint8_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svuint8_t" - }, "Architectures": [ "A64" ] @@ -163197,15 +163197,15 @@ "uint64_t imm_index", "svfloat16_t x" ], + "return_type": { + "value": "svfloat16x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svfloat16x2_t" - }, "Architectures": [ "A64" ] @@ -163218,15 +163218,15 @@ "uint64_t imm_index", "svfloat32_t x" ], + "return_type": { + "value": "svfloat32x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svfloat32x2_t" - }, "Architectures": [ "A64" ] @@ -163239,15 +163239,15 @@ "uint64_t imm_index", "svfloat64_t x" ], + "return_type": { + "value": "svfloat64x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svfloat64x2_t" - }, "Architectures": [ "A64" ] @@ -163260,15 +163260,15 @@ "uint64_t imm_index", "svint16_t x" ], + "return_type": { + "value": "svint16x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svint16x2_t" - }, "Architectures": [ "A64" ] @@ -163281,15 +163281,15 @@ "uint64_t imm_index", "svint32_t x" ], + "return_type": { + "value": "svint32x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svint32x2_t" - }, "Architectures": [ "A64" ] @@ -163302,15 +163302,15 @@ "uint64_t imm_index", "svint64_t x" ], + "return_type": { + "value": "svint64x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svint64x2_t" - }, "Architectures": [ "A64" ] @@ -163323,15 +163323,15 @@ "uint64_t imm_index", "svint8_t x" ], + "return_type": { + "value": "svint8x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svint8x2_t" - }, "Architectures": [ "A64" ] @@ -163344,15 +163344,15 @@ "uint64_t imm_index", "svuint16_t x" ], + "return_type": { + "value": "svuint16x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svuint16x2_t" - }, "Architectures": [ "A64" ] @@ -163365,15 +163365,15 @@ "uint64_t imm_index", "svuint32_t x" ], + "return_type": { + "value": "svuint32x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svuint32x2_t" - }, "Architectures": [ "A64" ] @@ -163386,15 +163386,15 @@ "uint64_t imm_index", "svuint64_t x" ], + "return_type": { + "value": "svuint64x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svuint64x2_t" - }, "Architectures": [ "A64" ] @@ -163407,15 +163407,15 @@ "uint64_t imm_index", "svuint8_t x" ], + "return_type": { + "value": "svuint8x2_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 1 } }, - "return_type": { - "value": "svuint8x2_t" - }, "Architectures": [ "A64" ] @@ -163428,15 +163428,15 @@ "uint64_t imm_index", "svfloat16_t x" ], + "return_type": { + "value": "svfloat16x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svfloat16x3_t" - }, "Architectures": [ "A64" ] @@ -163449,15 +163449,15 @@ "uint64_t imm_index", "svfloat32_t x" ], + "return_type": { + "value": "svfloat32x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svfloat32x3_t" - }, "Architectures": [ "A64" ] @@ -163470,15 +163470,15 @@ "uint64_t imm_index", "svfloat64_t x" ], + "return_type": { + "value": "svfloat64x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svfloat64x3_t" - }, "Architectures": [ "A64" ] @@ -163491,15 +163491,15 @@ "uint64_t imm_index", "svint16_t x" ], + "return_type": { + "value": "svint16x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svint16x3_t" - }, "Architectures": [ "A64" ] @@ -163512,15 +163512,15 @@ "uint64_t imm_index", "svint32_t x" ], + "return_type": { + "value": "svint32x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svint32x3_t" - }, "Architectures": [ "A64" ] @@ -163533,15 +163533,15 @@ "uint64_t imm_index", "svint64_t x" ], + "return_type": { + "value": "svint64x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svint64x3_t" - }, "Architectures": [ "A64" ] @@ -163554,15 +163554,15 @@ "uint64_t imm_index", "svint8_t x" ], + "return_type": { + "value": "svint8x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svint8x3_t" - }, "Architectures": [ "A64" ] @@ -163575,15 +163575,15 @@ "uint64_t imm_index", "svuint16_t x" ], + "return_type": { + "value": "svuint16x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svuint16x3_t" - }, "Architectures": [ "A64" ] @@ -163596,15 +163596,15 @@ "uint64_t imm_index", "svuint32_t x" ], + "return_type": { + "value": "svuint32x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svuint32x3_t" - }, "Architectures": [ "A64" ] @@ -163617,15 +163617,15 @@ "uint64_t imm_index", "svuint64_t x" ], + "return_type": { + "value": "svuint64x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svuint64x3_t" - }, "Architectures": [ "A64" ] @@ -163638,15 +163638,15 @@ "uint64_t imm_index", "svuint8_t x" ], + "return_type": { + "value": "svuint8x3_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 2 } }, - "return_type": { - "value": "svuint8x3_t" - }, "Architectures": [ "A64" ] @@ -163659,15 +163659,15 @@ "uint64_t imm_index", "svfloat16_t x" ], + "return_type": { + "value": "svfloat16x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svfloat16x4_t" - }, "Architectures": [ "A64" ] @@ -163680,15 +163680,15 @@ "uint64_t imm_index", "svfloat32_t x" ], + "return_type": { + "value": "svfloat32x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svfloat32x4_t" - }, "Architectures": [ "A64" ] @@ -163701,15 +163701,15 @@ "uint64_t imm_index", "svfloat64_t x" ], + "return_type": { + "value": "svfloat64x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svfloat64x4_t" - }, "Architectures": [ "A64" ] @@ -163722,15 +163722,15 @@ "uint64_t imm_index", "svint16_t x" ], + "return_type": { + "value": "svint16x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svint16x4_t" - }, "Architectures": [ "A64" ] @@ -163743,15 +163743,15 @@ "uint64_t imm_index", "svint32_t x" ], + "return_type": { + "value": "svint32x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svint32x4_t" - }, "Architectures": [ "A64" ] @@ -163764,15 +163764,15 @@ "uint64_t imm_index", "svint64_t x" ], + "return_type": { + "value": "svint64x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svint64x4_t" - }, "Architectures": [ "A64" ] @@ -163785,15 +163785,15 @@ "uint64_t imm_index", "svint8_t x" ], + "return_type": { + "value": "svint8x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svint8x4_t" - }, "Architectures": [ "A64" ] @@ -163806,15 +163806,15 @@ "uint64_t imm_index", "svuint16_t x" ], + "return_type": { + "value": "svuint16x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svuint16x4_t" - }, "Architectures": [ "A64" ] @@ -163827,15 +163827,15 @@ "uint64_t imm_index", "svuint32_t x" ], + "return_type": { + "value": "svuint32x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svuint32x4_t" - }, "Architectures": [ "A64" ] @@ -163848,15 +163848,15 @@ "uint64_t imm_index", "svuint64_t x" ], + "return_type": { + "value": "svuint64x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svuint64x4_t" - }, "Architectures": [ "A64" ] @@ -163869,15 +163869,15 @@ "uint64_t imm_index", "svuint8_t x" ], + "return_type": { + "value": "svuint8x4_t" + }, "Arguments_Preparation": { "imm_index": { "minimum": 0, "maximum": 3 } }, - "return_type": { - "value": "svuint8x4_t" - }, "Architectures": [ "A64" ] @@ -201117,6 +201117,462 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vbfdot_f32", + "arguments": [ + "float32x2_t r", + "bfloat16x4_t a", + "bfloat16x4_t b" + ], + "return_type": { + "value": "float32x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.4H" + }, + "b": { + "register": "Vm.4H" + }, + "r": { + "register": "Vd.2S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFDOT" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfdot_lane_f32", + "arguments": [ + "float32x2_t r", + "bfloat16x4_t a", + "bfloat16x4_t b", + "const int lane" + ], + "return_type": { + "value": "float32x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.4H" + }, + "b": { + "register": "Vm.4H" + }, + "lane": { + "minimum": 0, + "maximum": 1 + }, + "r": { + "register": "Vd.2S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFDOT" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfdot_laneq_f32", + "arguments": [ + "float32x2_t r", + "bfloat16x4_t a", + "bfloat16x8_t b", + "const int lane" + ], + "return_type": { + "value": "float32x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.4H" + }, + "b": { + "register": "Vm.8H" + }, + "lane": { + "minimum": 0, + "maximum": 3 + }, + "r": { + "register": "Vd.2S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFDOT" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfdotq_f32", + "arguments": [ + "float32x4_t r", + "bfloat16x8_t a", + "bfloat16x8_t b" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + }, + "b": { + "register": "Vm.8H" + }, + "r": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFDOT" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfdotq_lane_f32", + "arguments": [ + "float32x4_t r", + "bfloat16x8_t a", + "bfloat16x4_t b", + "const int lane" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + }, + "b": { + "register": "Vm.4H" + }, + "lane": { + "minimum": 0, + "maximum": 1 + }, + "r": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFDOT" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfdotq_laneq_f32", + "arguments": [ + "float32x4_t r", + "bfloat16x8_t a", + "bfloat16x8_t b", + "const int lane" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + }, + "b": { + "register": "Vm.8H" + }, + "lane": { + "minimum": 0, + "maximum": 3 + }, + "r": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFDOT" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfmlalbq_f32", + "arguments": [ + "float32x4_t r", + "bfloat16x8_t a", + "bfloat16x8_t b" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + }, + "b": { + "register": "Vm.8H" + }, + "r": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFMLALB" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfmlalbq_lane_f32", + "arguments": [ + "float32x4_t r", + "bfloat16x8_t a", + "bfloat16x4_t b", + "const int lane" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + }, + "b": { + "register": "Vm.4H" + }, + "lane": { + "minimum": 0, + "maximum": 3 + }, + "r": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFMLALB" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfmlalbq_laneq_f32", + "arguments": [ + "float32x4_t r", + "bfloat16x8_t a", + "bfloat16x8_t b", + "const int lane" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + }, + "b": { + "register": "Vm.8H" + }, + "lane": { + "minimum": 0, + "maximum": 7 + }, + "r": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFMLALB" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfmlaltq_f32", + "arguments": [ + "float32x4_t r", + "bfloat16x8_t a", + "bfloat16x8_t b" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + }, + "b": { + "register": "Vm.8H" + }, + "r": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFMLALT" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfmlaltq_lane_f32", + "arguments": [ + "float32x4_t r", + "bfloat16x8_t a", + "bfloat16x4_t b", + "const int lane" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + }, + "b": { + "register": "Vm.4H" + }, + "lane": { + "minimum": 0, + "maximum": 3 + }, + "r": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFMLALT" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfmlaltq_laneq_f32", + "arguments": [ + "float32x4_t r", + "bfloat16x8_t a", + "bfloat16x8_t b", + "const int lane" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + }, + "b": { + "register": "Vm.8H" + }, + "lane": { + "minimum": 0, + "maximum": 7 + }, + "r": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFMLALT" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vbfmmlaq_f32", + "arguments": [ + "float32x4_t r", + "bfloat16x8_t a", + "bfloat16x8_t b" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + }, + "b": { + "register": "Vm.8H" + }, + "r": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFMMLA" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vbic_s16", @@ -212637,6 +213093,35 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcombine_bf16", + "arguments": [ + "bfloat16x4_t low", + "bfloat16x4_t high" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "high": { + "register": "Vm.4H" + }, + "low": { + "register": "Vn.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP", + "INS" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcombine_f16", @@ -213054,6 +213539,43 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcopy_lane_bf16", + "arguments": [ + "bfloat16x4_t a", + "const int lane1", + "bfloat16x4_t b", + "const int lane2" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + }, + "b": { + "register": "Vn.4H" + }, + "lane1": { + "minimum": 0, + "maximum": 3 + }, + "lane2": { + "minimum": 0, + "maximum": 3 + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "INS" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcopy_lane_f32", @@ -213536,6 +214058,43 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcopy_laneq_bf16", + "arguments": [ + "bfloat16x4_t a", + "const int lane1", + "bfloat16x8_t b", + "const int lane2" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + }, + "b": { + "register": "Vn.8H" + }, + "lane1": { + "minimum": 0, + "maximum": 3 + }, + "lane2": { + "minimum": 0, + "maximum": 7 + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "INS" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcopy_laneq_f32", @@ -214018,6 +214577,43 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcopyq_lane_bf16", + "arguments": [ + "bfloat16x8_t a", + "const int lane1", + "bfloat16x4_t b", + "const int lane2" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + }, + "b": { + "register": "Vn.4H" + }, + "lane1": { + "minimum": 0, + "maximum": 7 + }, + "lane2": { + "minimum": 0, + "maximum": 3 + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "INS" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcopyq_lane_f32", @@ -214500,6 +215096,43 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcopyq_laneq_bf16", + "arguments": [ + "bfloat16x8_t a", + "const int lane1", + "bfloat16x8_t b", + "const int lane2" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + }, + "b": { + "register": "Vn.8H" + }, + "lane1": { + "minimum": 0, + "maximum": 7 + }, + "lane2": { + "minimum": 0, + "maximum": 7 + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "INS" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcopyq_laneq_f32", @@ -214982,6 +215615,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcreate_bf16", + "arguments": [ + "uint64_t a" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "INS" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcreate_f16", @@ -215329,6 +215986,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcvt_bf16_f32", + "arguments": [ + "float32x4_t a" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFCVTN" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcvt_f16_f32", @@ -215402,6 +216083,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcvt_f32_bf16", + "arguments": [ + "bfloat16x4_t a" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "SHLL" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcvt_f32_f16", @@ -216395,6 +217100,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcvtah_f32_bf16", + "arguments": [ + "bfloat16_t a" + ], + "return_type": { + "value": "float32_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Hn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "SHL" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcvtah_s16_f16", @@ -217017,6 +217746,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcvth_bf16_f32", + "arguments": [ + "float32_t a" + ], + "return_type": { + "value": "bfloat16_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Sn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFCVT" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcvth_f16_s16", @@ -219597,6 +220350,106 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vcvtq_high_bf16_f32", + "arguments": [ + "bfloat16x8_t inactive", + "float32x4_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.4S" + }, + "inactive": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFCVTN2" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vcvtq_high_f32_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "SHLL2" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vcvtq_low_bf16_f32", + "arguments": [ + "float32x4_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "BFCVTN" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vcvtq_low_f32_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "float32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "SHLL" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vcvtq_n_f16_s16", @@ -221026,6 +221879,35 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vdup_lane_bf16", + "arguments": [ + "bfloat16x4_t vec", + "const int lane" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "vec": { + "register": "Vn.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vdup_lane_f16", @@ -221443,6 +222325,35 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vdup_laneq_bf16", + "arguments": [ + "bfloat16x8_t vec", + "const int lane" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "vec": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vdup_laneq_f16", @@ -221835,6 +222746,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vdup_n_bf16", + "arguments": [ + "bfloat16_t value" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "value": { + "register": "rn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vdup_n_f16", @@ -222518,6 +223453,35 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vduph_lane_bf16", + "arguments": [ + "bfloat16x4_t vec", + "const int lane" + ], + "return_type": { + "value": "bfloat16_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "vec": { + "register": "Vn.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vduph_lane_f16", @@ -222630,6 +223594,35 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vduph_laneq_bf16", + "arguments": [ + "bfloat16x8_t vec", + "const int lane" + ], + "return_type": { + "value": "bfloat16_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "vec": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vduph_laneq_f16", @@ -222742,6 +223735,35 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vdupq_lane_bf16", + "arguments": [ + "bfloat16x4_t vec", + "const int lane" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "vec": { + "register": "Vn.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vdupq_lane_f16", @@ -223159,6 +224181,35 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vdupq_laneq_bf16", + "arguments": [ + "bfloat16x8_t vec", + "const int lane" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "vec": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vdupq_laneq_f16", @@ -223551,6 +224602,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vdupq_n_bf16", + "arguments": [ + "bfloat16_t value" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "value": { + "register": "rn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vdupq_n_f16", @@ -228692,6 +229767,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vget_high_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vget_high_f16", @@ -229041,13 +230140,13 @@ }, { "SIMD_ISA": "Neon", - "name": "vget_lane_f16", + "name": "vget_lane_bf16", "arguments": [ - "float16x4_t v", + "bfloat16x4_t v", "const int lane" ], "return_type": { - "value": "float16_t" + "value": "bfloat16_t" }, "Arguments_Preparation": { "lane": { @@ -229059,7 +230158,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -229071,160 +230169,13 @@ }, { "SIMD_ISA": "Neon", - "name": "vget_lane_f32", - "arguments": [ - "float32x2_t v", - "const int lane" - ], - "return_type": { - "value": "float32_t" - }, - "Arguments_Preparation": { - "lane": { - "minimum": 0, - "maximum": 1 - }, - "v": { - "register": "Vn.2S" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "DUP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vget_lane_f64", - "arguments": [ - "float64x1_t v", - "const int lane" - ], - "return_type": { - "value": "float64_t" - }, - "Arguments_Preparation": { - "lane": { - "minimum": 0, - "maximum": 0 - }, - "v": { - "register": "Vn.1D" - } - }, - "Architectures": [ - "A64" - ], - "instructions": [ - [ - "DUP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vget_lane_p16", - "arguments": [ - "poly16x4_t v", - "const int lane" - ], - "return_type": { - "value": "poly16_t" - }, - "Arguments_Preparation": { - "lane": { - "minimum": 0, - "maximum": 3 - }, - "v": { - "register": "Vn.4H" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "UMOV" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vget_lane_p64", - "arguments": [ - "poly64x1_t v", - "const int lane" - ], - "return_type": { - "value": "poly64_t" - }, - "Arguments_Preparation": { - "lane": { - "minimum": 0, - "maximum": 0 - }, - "v": { - "register": "Vn.1D" - } - }, - "Architectures": [ - "A32", - "A64" - ], - "instructions": [ - [ - "UMOV" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vget_lane_p8", - "arguments": [ - "poly8x8_t v", - "const int lane" - ], - "return_type": { - "value": "poly8_t" - }, - "Arguments_Preparation": { - "lane": { - "minimum": 0, - "maximum": 7 - }, - "v": { - "register": "Vn.8B" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "UMOV" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vget_lane_s16", + "name": "vget_lane_f16", "arguments": [ - "int16x4_t v", + "float16x4_t v", "const int lane" ], "return_type": { - "value": "int16_t" + "value": "float16_t" }, "Arguments_Preparation": { "lane": { @@ -229242,19 +230193,19 @@ ], "instructions": [ [ - "SMOV" + "DUP" ] ] }, { "SIMD_ISA": "Neon", - "name": "vget_lane_s32", + "name": "vget_lane_f32", "arguments": [ - "int32x2_t v", + "float32x2_t v", "const int lane" ], "return_type": { - "value": "int32_t" + "value": "float32_t" }, "Arguments_Preparation": { "lane": { @@ -229272,19 +230223,19 @@ ], "instructions": [ [ - "SMOV" + "DUP" ] ] }, { "SIMD_ISA": "Neon", - "name": "vget_lane_s64", + "name": "vget_lane_f64", "arguments": [ - "int64x1_t v", + "float64x1_t v", "const int lane" ], "return_type": { - "value": "int64_t" + "value": "float64_t" }, "Arguments_Preparation": { "lane": { @@ -229296,55 +230247,23 @@ } }, "Architectures": [ - "v7", - "A32", "A64" ], "instructions": [ [ - "UMOV" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vget_lane_s8", - "arguments": [ - "int8x8_t v", - "const int lane" - ], - "return_type": { - "value": "int8_t" - }, - "Arguments_Preparation": { - "lane": { - "minimum": 0, - "maximum": 7 - }, - "v": { - "register": "Vn.8B" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "SMOV" + "DUP" ] ] }, { "SIMD_ISA": "Neon", - "name": "vget_lane_u16", + "name": "vget_lane_p16", "arguments": [ - "uint16x4_t v", + "poly16x4_t v", "const int lane" ], "return_type": { - "value": "uint16_t" + "value": "poly16_t" }, "Arguments_Preparation": { "lane": { @@ -229368,43 +230287,13 @@ }, { "SIMD_ISA": "Neon", - "name": "vget_lane_u32", - "arguments": [ - "uint32x2_t v", - "const int lane" - ], - "return_type": { - "value": "uint32_t" - }, - "Arguments_Preparation": { - "lane": { - "minimum": 0, - "maximum": 1 - }, - "v": { - "register": "Vn.2S" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "UMOV" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vget_lane_u64", + "name": "vget_lane_p64", "arguments": [ - "uint64x1_t v", + "poly64x1_t v", "const int lane" ], "return_type": { - "value": "uint64_t" + "value": "poly64_t" }, "Arguments_Preparation": { "lane": { @@ -229416,7 +230305,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -229428,13 +230316,13 @@ }, { "SIMD_ISA": "Neon", - "name": "vget_lane_u8", + "name": "vget_lane_p8", "arguments": [ - "uint8x8_t v", + "poly8x8_t v", "const int lane" ], "return_type": { - "value": "uint8_t" + "value": "poly8_t" }, "Arguments_Preparation": { "lane": { @@ -229456,6 +230344,270 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vget_lane_s16", + "arguments": [ + "int16x4_t v", + "const int lane" + ], + "return_type": { + "value": "int16_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "v": { + "register": "Vn.4H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "SMOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vget_lane_s32", + "arguments": [ + "int32x2_t v", + "const int lane" + ], + "return_type": { + "value": "int32_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 1 + }, + "v": { + "register": "Vn.2S" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "SMOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vget_lane_s64", + "arguments": [ + "int64x1_t v", + "const int lane" + ], + "return_type": { + "value": "int64_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 0 + }, + "v": { + "register": "Vn.1D" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "UMOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vget_lane_s8", + "arguments": [ + "int8x8_t v", + "const int lane" + ], + "return_type": { + "value": "int8_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "v": { + "register": "Vn.8B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "SMOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vget_lane_u16", + "arguments": [ + "uint16x4_t v", + "const int lane" + ], + "return_type": { + "value": "uint16_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "v": { + "register": "Vn.4H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "UMOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vget_lane_u32", + "arguments": [ + "uint32x2_t v", + "const int lane" + ], + "return_type": { + "value": "uint32_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 1 + }, + "v": { + "register": "Vn.2S" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "UMOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vget_lane_u64", + "arguments": [ + "uint64x1_t v", + "const int lane" + ], + "return_type": { + "value": "uint64_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 0 + }, + "v": { + "register": "Vn.1D" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "UMOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vget_lane_u8", + "arguments": [ + "uint8x8_t v", + "const int lane" + ], + "return_type": { + "value": "uint8_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "v": { + "register": "Vn.8B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "UMOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vget_low_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vget_low_f16", @@ -229803,6 +230955,35 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vgetq_lane_bf16", + "arguments": [ + "bfloat16x8_t v", + "const int lane" + ], + "return_type": { + "value": "bfloat16_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "v": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "DUP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vgetq_lane_f16", @@ -230916,6 +232097,127 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld1_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1_bf16_x2", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4x2_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1_bf16_x3", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4x3_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1_bf16_x4", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4x4_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1_dup_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD1R" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld1_dup_f16", @@ -231557,14 +232859,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vld1_lane_f16", + "name": "vld1_lane_bf16", "arguments": [ - "float16_t const * ptr", - "float16x4_t src", + "bfloat16_t const * ptr", + "bfloat16x4_t src", "const int lane" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "lane": { @@ -231579,7 +232881,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -231591,25 +232892,25 @@ }, { "SIMD_ISA": "Neon", - "name": "vld1_lane_f32", + "name": "vld1_lane_f16", "arguments": [ - "float32_t const * ptr", - "float32x2_t src", + "float16_t const * ptr", + "float16x4_t src", "const int lane" ], "return_type": { - "value": "float32x2_t" + "value": "float16x4_t" }, "Arguments_Preparation": { "lane": { "minimum": 0, - "maximum": 1 + "maximum": 3 }, "ptr": { "register": "Xn" }, "src": { - "register": "Vt.2S" + "register": "Vt.4H" } }, "Architectures": [ @@ -231625,57 +232926,25 @@ }, { "SIMD_ISA": "Neon", - "name": "vld1_lane_f64", - "arguments": [ - "float64_t const * ptr", - "float64x1_t src", - "const int lane" - ], - "return_type": { - "value": "float64x1_t" - }, - "Arguments_Preparation": { - "lane": { - "minimum": 0, - "maximum": 0 - }, - "ptr": { - "register": "Xn" - }, - "src": { - "register": "Vt.1D" - } - }, - "Architectures": [ - "A64" - ], - "instructions": [ - [ - "LD1" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vld1_lane_p16", + "name": "vld1_lane_f32", "arguments": [ - "poly16_t const * ptr", - "poly16x4_t src", + "float32_t const * ptr", + "float32x2_t src", "const int lane" ], "return_type": { - "value": "poly16x4_t" + "value": "float32x2_t" }, "Arguments_Preparation": { "lane": { "minimum": 0, - "maximum": 3 + "maximum": 1 }, "ptr": { "register": "Xn" }, "src": { - "register": "Vt.4H" + "register": "Vt.2S" } }, "Architectures": [ @@ -231691,14 +232960,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vld1_lane_p64", + "name": "vld1_lane_f64", "arguments": [ - "poly64_t const * ptr", - "poly64x1_t src", + "float64_t const * ptr", + "float64x1_t src", "const int lane" ], "return_type": { - "value": "poly64x1_t" + "value": "float64x1_t" }, "Arguments_Preparation": { "lane": { @@ -231713,41 +232982,6 @@ } }, "Architectures": [ - "A32", - "A64" - ], - "instructions": [ - [ - "LD1" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vld1_lane_p8", - "arguments": [ - "poly8_t const * ptr", - "poly8x8_t src", - "const int lane" - ], - "return_type": { - "value": "poly8x8_t" - }, - "Arguments_Preparation": { - "lane": { - "minimum": 0, - "maximum": 7 - }, - "ptr": { - "register": "Xn" - }, - "src": { - "register": "Vt.8B" - } - }, - "Architectures": [ - "v7", - "A32", "A64" ], "instructions": [ @@ -231758,14 +232992,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vld1_lane_s16", + "name": "vld1_lane_p16", "arguments": [ - "int16_t const * ptr", - "int16x4_t src", + "poly16_t const * ptr", + "poly16x4_t src", "const int lane" ], "return_type": { - "value": "int16x4_t" + "value": "poly16x4_t" }, "Arguments_Preparation": { "lane": { @@ -231792,48 +233026,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vld1_lane_s32", - "arguments": [ - "int32_t const * ptr", - "int32x2_t src", - "const int lane" - ], - "return_type": { - "value": "int32x2_t" - }, - "Arguments_Preparation": { - "lane": { - "minimum": 0, - "maximum": 1 - }, - "ptr": { - "register": "Xn" - }, - "src": { - "register": "Vt.2S" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "LD1" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vld1_lane_s64", + "name": "vld1_lane_p64", "arguments": [ - "int64_t const * ptr", - "int64x1_t src", + "poly64_t const * ptr", + "poly64x1_t src", "const int lane" ], "return_type": { - "value": "int64x1_t" + "value": "poly64x1_t" }, "Arguments_Preparation": { "lane": { @@ -231848,7 +233048,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -231860,14 +233059,150 @@ }, { "SIMD_ISA": "Neon", - "name": "vld1_lane_s8", + "name": "vld1_lane_p8", "arguments": [ - "int8_t const * ptr", - "int8x8_t src", + "poly8_t const * ptr", + "poly8x8_t src", "const int lane" ], "return_type": { - "value": "int8x8_t" + "value": "poly8x8_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "ptr": { + "register": "Xn" + }, + "src": { + "register": "Vt.8B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1_lane_s16", + "arguments": [ + "int16_t const * ptr", + "int16x4_t src", + "const int lane" + ], + "return_type": { + "value": "int16x4_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "ptr": { + "register": "Xn" + }, + "src": { + "register": "Vt.4H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1_lane_s32", + "arguments": [ + "int32_t const * ptr", + "int32x2_t src", + "const int lane" + ], + "return_type": { + "value": "int32x2_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 1 + }, + "ptr": { + "register": "Xn" + }, + "src": { + "register": "Vt.2S" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1_lane_s64", + "arguments": [ + "int64_t const * ptr", + "int64x1_t src", + "const int lane" + ], + "return_type": { + "value": "int64x1_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 0 + }, + "ptr": { + "register": "Xn" + }, + "src": { + "register": "Vt.1D" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1_lane_s8", + "arguments": [ + "int8_t const * ptr", + "int8x8_t src", + "const int lane" + ], + "return_type": { + "value": "int8x8_t" }, "Arguments_Preparation": { "lane": { @@ -233124,6 +234459,127 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld1q_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1q_bf16_x2", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8x2_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1q_bf16_x3", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8x3_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1q_bf16_x4", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8x4_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld1q_dup_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD1R" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld1q_dup_f16", @@ -233763,6 +235219,39 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld1q_lane_bf16", + "arguments": [ + "bfloat16_t const * ptr", + "bfloat16x8_t src", + "const int lane" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "ptr": { + "register": "Xn" + }, + "src": { + "register": "Vt.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD1" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld1q_lane_f16", @@ -235332,6 +236821,54 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld2_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4x2_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD2" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld2_dup_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4x2_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD2R" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld2_dup_f16", @@ -235752,6 +237289,42 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld2_lane_bf16", + "arguments": [ + "bfloat16_t const * ptr", + "bfloat16x4x2_t src", + "const int lane" + ], + "return_type": { + "value": "bfloat16x4x2_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "ptr": { + "register": "Xn" + }, + "src.val[0]": { + "register": "Vt.4H" + }, + "src.val[1]": { + "register": "Vt2.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD2" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld2_lane_f16", @@ -236536,6 +238109,54 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld2q_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8x2_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD2" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld2q_dup_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8x2_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD2R" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld2q_dup_f16", @@ -236951,6 +238572,42 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld2q_lane_bf16", + "arguments": [ + "bfloat16_t const * ptr", + "bfloat16x8x2_t src", + "const int lane" + ], + "return_type": { + "value": "bfloat16x8x2_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "ptr": { + "register": "Xn" + }, + "src.val[0]": { + "register": "Vt.8H" + }, + "src.val[1]": { + "register": "Vt2.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD2" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld2q_lane_f16", @@ -237724,6 +239381,54 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld3_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4x3_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD3" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld3_dup_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4x3_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD3R" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld3_dup_f16", @@ -238144,6 +239849,45 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld3_lane_bf16", + "arguments": [ + "bfloat16_t const * ptr", + "bfloat16x4x3_t src", + "const int lane" + ], + "return_type": { + "value": "bfloat16x4x3_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "ptr": { + "register": "Xn" + }, + "src.val[0]": { + "register": "Vt.4H" + }, + "src.val[1]": { + "register": "Vt2.4H" + }, + "src.val[2]": { + "register": "Vt3.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD3" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld3_lane_f16", @@ -238970,6 +240714,54 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld3q_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8x3_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD3" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld3q_dup_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8x3_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD3R" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld3q_dup_f16", @@ -239385,6 +241177,45 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld3q_lane_bf16", + "arguments": [ + "bfloat16_t const * ptr", + "bfloat16x8x3_t src", + "const int lane" + ], + "return_type": { + "value": "bfloat16x8x3_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "ptr": { + "register": "Xn" + }, + "src.val[0]": { + "register": "Vt.8H" + }, + "src.val[1]": { + "register": "Vt2.8H" + }, + "src.val[2]": { + "register": "Vt3.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD3" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld3q_lane_f16", @@ -240200,6 +242031,54 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld4_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4x4_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD4" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld4_dup_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x4x4_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD4R" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld4_dup_f16", @@ -240620,6 +242499,48 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld4_lane_bf16", + "arguments": [ + "bfloat16_t const * ptr", + "bfloat16x4x4_t src", + "const int lane" + ], + "return_type": { + "value": "bfloat16x4x4_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "ptr": { + "register": "Xn" + }, + "src.val[0]": { + "register": "Vt.4H" + }, + "src.val[1]": { + "register": "Vt2.4H" + }, + "src.val[2]": { + "register": "Vt3.4H" + }, + "src.val[3]": { + "register": "Vt4.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD4" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld4_lane_f16", @@ -241488,6 +243409,54 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld4q_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8x4_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD4" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vld4q_dup_bf16", + "arguments": [ + "bfloat16_t const * ptr" + ], + "return_type": { + "value": "bfloat16x8x4_t" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD4R" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld4q_dup_f16", @@ -241903,6 +243872,48 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vld4q_lane_bf16", + "arguments": [ + "bfloat16_t const * ptr", + "bfloat16x8x4_t src", + "const int lane" + ], + "return_type": { + "value": "bfloat16x8x4_t" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "ptr": { + "register": "Xn" + }, + "src.val[0]": { + "register": "Vt.8H" + }, + "src.val[1]": { + "register": "Vt2.8H" + }, + "src.val[2]": { + "register": "Vt3.8H" + }, + "src.val[3]": { + "register": "Vt4.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "LD4" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vld4q_lane_f16", @@ -243042,14 +245053,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vluti2_lane_f16", + "name": "vluti2_lane_bf16", "arguments": [ - "float16x4_t vn", + "bfloat16x4_t vn", "uint8x8_t vm", "const int index" ], "return_type": { - "value": "float16x8_t" + "value": "bfloat16x8_t" }, "Arguments_Preparation": { "index": { @@ -243074,14 +245085,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vluti2_lane_p16", + "name": "vluti2_lane_f16", "arguments": [ - "poly16x4_t vn", + "float16x4_t vn", "uint8x8_t vm", "const int index" ], "return_type": { - "value": "poly16x8_t" + "value": "float16x8_t" }, "Arguments_Preparation": { "index": { @@ -243106,46 +245117,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vluti2_lane_p8", - "arguments": [ - "poly8x8_t vn", - "uint8x8_t vm", - "const int index" - ], - "return_type": { - "value": "poly8x16_t" - }, - "Arguments_Preparation": { - "index": { - "minimum": 0, - "maximum": 1 - }, - "vm": { - "register": "Vm" - }, - "vn": { - "register": "Vn.16B" - } - }, - "Architectures": [ - "A64" - ], - "instructions": [ - [ - "LUTI2" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vluti2_lane_s16", + "name": "vluti2_lane_p16", "arguments": [ - "int16x4_t vn", + "poly16x4_t vn", "uint8x8_t vm", "const int index" ], "return_type": { - "value": "int16x8_t" + "value": "poly16x8_t" }, "Arguments_Preparation": { "index": { @@ -243170,14 +245149,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vluti2_lane_s8", + "name": "vluti2_lane_p8", "arguments": [ - "int8x8_t vn", + "poly8x8_t vn", "uint8x8_t vm", "const int index" ], "return_type": { - "value": "int8x16_t" + "value": "poly8x16_t" }, "Arguments_Preparation": { "index": { @@ -243202,14 +245181,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vluti2_lane_u16", + "name": "vluti2_lane_s16", "arguments": [ - "uint16x4_t vn", + "int16x4_t vn", "uint8x8_t vm", "const int index" ], "return_type": { - "value": "uint16x8_t" + "value": "int16x8_t" }, "Arguments_Preparation": { "index": { @@ -243234,14 +245213,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vluti2_lane_u8", + "name": "vluti2_lane_s8", "arguments": [ - "uint8x8_t vn", + "int8x8_t vn", "uint8x8_t vm", "const int index" ], "return_type": { - "value": "uint8x16_t" + "value": "int8x16_t" }, "Arguments_Preparation": { "index": { @@ -243264,6 +245243,102 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vluti2_lane_u16", + "arguments": [ + "uint16x4_t vn", + "uint8x8_t vm", + "const int index" + ], + "return_type": { + "value": "uint16x8_t" + }, + "Arguments_Preparation": { + "index": { + "minimum": 0, + "maximum": 3 + }, + "vm": { + "register": "Vm" + }, + "vn": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "LUTI2" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vluti2_lane_u8", + "arguments": [ + "uint8x8_t vn", + "uint8x8_t vm", + "const int index" + ], + "return_type": { + "value": "uint8x16_t" + }, + "Arguments_Preparation": { + "index": { + "minimum": 0, + "maximum": 1 + }, + "vm": { + "register": "Vm" + }, + "vn": { + "register": "Vn.16B" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "LUTI2" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vluti2_laneq_bf16", + "arguments": [ + "bfloat16x4_t vn", + "uint8x16_t vm", + "const int index" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "index": { + "minimum": 0, + "maximum": 7 + }, + "vm": { + "register": "Vm" + }, + "vn": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "LUTI2" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vluti2_laneq_f16", @@ -243488,6 +245563,38 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vluti2q_lane_bf16", + "arguments": [ + "bfloat16x8_t vn", + "uint8x8_t vm", + "const int index" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "index": { + "minimum": 0, + "maximum": 3 + }, + "vm": { + "register": "Vm" + }, + "vn": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "LUTI2" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vluti2q_lane_f16", @@ -243712,6 +245819,38 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vluti2q_laneq_bf16", + "arguments": [ + "bfloat16x8_t vn", + "uint8x16_t vm", + "const int index" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "index": { + "minimum": 0, + "maximum": 7 + }, + "vm": { + "register": "Vm" + }, + "vn": { + "register": "Vn.8H" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "LUTI2" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vluti2q_laneq_f16", @@ -243938,49 +246077,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vluti4q_lane_f16_x2", - "arguments": [ - "float16x8x2_t vn", - "uint8x8_t vm", - "const int index" - ], - "return_type": { - "value": "float16x8_t" - }, - "Arguments_Preparation": { - "index": { - "minimum": 0, - "maximum": 1 - }, - "vm": { - "register": "Vm" - }, - "vn.val[0]": { - "register": "Vn1.8H" - }, - "vn.val[1]": { - "register": "Vn2.8H" - } - }, - "Architectures": [ - "A64" - ], - "instructions": [ - [ - "LUTI4" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vluti4q_lane_p16_x2", + "name": "vluti4q_lane_bf16_x2", "arguments": [ - "poly16x8x2_t vn", + "bfloat16x8x2_t vn", "uint8x8_t vm", "const int index" ], "return_type": { - "value": "poly16x8_t" + "value": "bfloat16x8_t" }, "Arguments_Preparation": { "index": { @@ -244008,46 +246112,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vluti4q_lane_p8", - "arguments": [ - "poly8x16_t vn", - "uint8x8_t vm", - "const int index" - ], - "return_type": { - "value": "poly8x16_t" - }, - "Arguments_Preparation": { - "index": { - "minimum": 0, - "maximum": 0 - }, - "vm": { - "register": "Vm" - }, - "vn": { - "register": "Vn.16B" - } - }, - "Architectures": [ - "A64" - ], - "instructions": [ - [ - "LUTI4" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vluti4q_lane_s16_x2", + "name": "vluti4q_lane_f16_x2", "arguments": [ - "int16x8x2_t vn", + "float16x8x2_t vn", "uint8x8_t vm", "const int index" ], "return_type": { - "value": "int16x8_t" + "value": "float16x8_t" }, "Arguments_Preparation": { "index": { @@ -244075,46 +246147,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vluti4q_lane_s8", - "arguments": [ - "int8x16_t vn", - "uint8x8_t vm", - "const int index" - ], - "return_type": { - "value": "int8x16_t" - }, - "Arguments_Preparation": { - "index": { - "minimum": 0, - "maximum": 0 - }, - "vm": { - "register": "Vm" - }, - "vn": { - "register": "Vn.16B" - } - }, - "Architectures": [ - "A64" - ], - "instructions": [ - [ - "LUTI4" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vluti4q_lane_u16_x2", + "name": "vluti4q_lane_p16_x2", "arguments": [ - "uint16x8x2_t vn", + "poly16x8x2_t vn", "uint8x8_t vm", "const int index" ], "return_type": { - "value": "uint16x8_t" + "value": "poly16x8_t" }, "Arguments_Preparation": { "index": { @@ -244142,14 +246182,14 @@ }, { "SIMD_ISA": "Neon", - "name": "vluti4q_lane_u8", + "name": "vluti4q_lane_p8", "arguments": [ - "uint8x16_t vn", + "poly8x16_t vn", "uint8x8_t vm", "const int index" ], "return_type": { - "value": "uint8x16_t" + "value": "poly8x16_t" }, "Arguments_Preparation": { "index": { @@ -244172,6 +246212,175 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vluti4q_lane_s16_x2", + "arguments": [ + "int16x8x2_t vn", + "uint8x8_t vm", + "const int index" + ], + "return_type": { + "value": "int16x8_t" + }, + "Arguments_Preparation": { + "index": { + "minimum": 0, + "maximum": 1 + }, + "vm": { + "register": "Vm" + }, + "vn.val[0]": { + "register": "Vn1.8H" + }, + "vn.val[1]": { + "register": "Vn2.8H" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "LUTI4" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vluti4q_lane_s8", + "arguments": [ + "int8x16_t vn", + "uint8x8_t vm", + "const int index" + ], + "return_type": { + "value": "int8x16_t" + }, + "Arguments_Preparation": { + "index": { + "minimum": 0, + "maximum": 0 + }, + "vm": { + "register": "Vm" + }, + "vn": { + "register": "Vn.16B" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "LUTI4" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vluti4q_lane_u16_x2", + "arguments": [ + "uint16x8x2_t vn", + "uint8x8_t vm", + "const int index" + ], + "return_type": { + "value": "uint16x8_t" + }, + "Arguments_Preparation": { + "index": { + "minimum": 0, + "maximum": 1 + }, + "vm": { + "register": "Vm" + }, + "vn.val[0]": { + "register": "Vn1.8H" + }, + "vn.val[1]": { + "register": "Vn2.8H" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "LUTI4" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vluti4q_lane_u8", + "arguments": [ + "uint8x16_t vn", + "uint8x8_t vm", + "const int index" + ], + "return_type": { + "value": "uint8x16_t" + }, + "Arguments_Preparation": { + "index": { + "minimum": 0, + "maximum": 0 + }, + "vm": { + "register": "Vm" + }, + "vn": { + "register": "Vn.16B" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "LUTI4" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vluti4q_laneq_bf16_x2", + "arguments": [ + "bfloat16x8x2_t vn", + "uint8x16_t vm", + "const int index" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "index": { + "minimum": 0, + "maximum": 3 + }, + "vm": { + "register": "Vm" + }, + "vn.val[0]": { + "register": "Vn1.8H" + }, + "vn.val[1]": { + "register": "Vn2.8H" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "LUTI4" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vluti4q_laneq_f16_x2", @@ -276421,12 +278630,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_f32", + "name": "vreinterpret_bf16_f32", "arguments": [ "float32x2_t a" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "a": { @@ -276434,7 +278643,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -276446,12 +278654,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_f64", + "name": "vreinterpret_bf16_f64", "arguments": [ "float64x1_t a" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "a": { @@ -276469,12 +278677,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_p16", + "name": "vreinterpret_bf16_p16", "arguments": [ "poly16x4_t a" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "a": { @@ -276482,7 +278690,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -276494,12 +278701,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_p64", + "name": "vreinterpret_bf16_p64", "arguments": [ "poly64x1_t a" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "a": { @@ -276518,12 +278725,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_p8", + "name": "vreinterpret_bf16_p8", "arguments": [ "poly8x8_t a" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "a": { @@ -276531,7 +278738,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -276543,12 +278749,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_s16", + "name": "vreinterpret_bf16_s16", "arguments": [ "int16x4_t a" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "a": { @@ -276556,7 +278762,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -276568,12 +278773,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_s32", + "name": "vreinterpret_bf16_s32", "arguments": [ "int32x2_t a" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "a": { @@ -276581,7 +278786,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -276593,12 +278797,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_s64", + "name": "vreinterpret_bf16_s64", "arguments": [ "int64x1_t a" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "a": { @@ -276606,7 +278810,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -276618,12 +278821,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_s8", + "name": "vreinterpret_bf16_s8", "arguments": [ "int8x8_t a" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "a": { @@ -276631,7 +278834,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -276643,12 +278845,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_u16", + "name": "vreinterpret_bf16_u16", "arguments": [ "uint16x4_t a" ], "return_type": { - "value": "float16x4_t" + "value": "bfloat16x4_t" }, "Arguments_Preparation": { "a": { @@ -276656,7 +278858,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -276668,7 +278869,326 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_f16_u32", + "name": "vreinterpret_bf16_u32", + "arguments": [ + "uint32x2_t a" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_bf16_u64", + "arguments": [ + "uint64x1_t a" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.1D" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_bf16_u8", + "arguments": [ + "uint8x8_t a" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8B" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_f32", + "arguments": [ + "float32x2_t a" + ], + "return_type": { + "value": "float16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2S" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_f64", + "arguments": [ + "float64x1_t a" + ], + "return_type": { + "value": "float16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.1D" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_p16", + "arguments": [ + "poly16x4_t a" + ], + "return_type": { + "value": "float16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_p64", + "arguments": [ + "poly64x1_t a" + ], + "return_type": { + "value": "float16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.1D" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_p8", + "arguments": [ + "poly8x8_t a" + ], + "return_type": { + "value": "float16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_s16", + "arguments": [ + "int16x4_t a" + ], + "return_type": { + "value": "float16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_s32", + "arguments": [ + "int32x2_t a" + ], + "return_type": { + "value": "float16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2S" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_s64", + "arguments": [ + "int64x1_t a" + ], + "return_type": { + "value": "float16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.1D" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_s8", + "arguments": [ + "int8x8_t a" + ], + "return_type": { + "value": "float16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_u16", + "arguments": [ + "uint16x4_t a" + ], + "return_type": { + "value": "float16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f16_u32", "arguments": [ "uint32x2_t a" ], @@ -276741,6 +279261,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f32_bf16", + "arguments": [ + "bfloat16x4_t a" + ], + "return_type": { + "value": "float32x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpret_f32_f16", @@ -277039,6 +279583,29 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_f64_bf16", + "arguments": [ + "bfloat16x4_t a" + ], + "return_type": { + "value": "float64x1_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpret_f64_f16", @@ -277338,6 +279905,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_p16_bf16", + "arguments": [ + "bfloat16x4_t a" + ], + "return_type": { + "value": "poly16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpret_p16_f16", @@ -277660,6 +280251,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_p64_bf16", + "arguments": [ + "bfloat16x4_t a" + ], + "return_type": { + "value": "poly64x1_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpret_p64_f16", @@ -277947,6 +280562,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_p8_bf16", + "arguments": [ + "bfloat16x4_t a" + ], + "return_type": { + "value": "poly8x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpret_p8_f16", @@ -278269,6 +280908,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_s16_bf16", + "arguments": [ + "bfloat16x4_t a" + ], + "return_type": { + "value": "int16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpret_s16_f16", @@ -278591,6 +281254,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_s32_bf16", + "arguments": [ + "bfloat16x4_t a" + ], + "return_type": { + "value": "int32x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpret_s32_f16", @@ -278913,6 +281600,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_s64_bf16", + "arguments": [ + "bfloat16x4_t a" + ], + "return_type": { + "value": "int64x1_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpret_s64_f16", @@ -279235,6 +281946,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_s8_bf16", + "arguments": [ + "bfloat16x4_t a" + ], + "return_type": { + "value": "int8x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpret_s8_f16", @@ -279559,9 +282294,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_f16", + "name": "vreinterpret_u16_bf16", "arguments": [ - "float16x4_t a" + "bfloat16x4_t a" ], "return_type": { "value": "uint16x4_t" @@ -279572,7 +282307,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -279584,16 +282318,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_f32", + "name": "vreinterpret_u16_f16", "arguments": [ - "float32x2_t a" + "float16x4_t a" ], "return_type": { "value": "uint16x4_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.2S" + "register": "Vd.4H" } }, "Architectures": [ @@ -279609,39 +282343,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_f64", - "arguments": [ - "float64x1_t a" - ], - "return_type": { - "value": "uint16x4_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.1D" - } - }, - "Architectures": [ - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_p16", + "name": "vreinterpret_u16_f32", "arguments": [ - "poly16x4_t a" + "float32x2_t a" ], "return_type": { "value": "uint16x4_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4H" + "register": "Vd.2S" } }, "Architectures": [ @@ -279657,9 +282368,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_p64", + "name": "vreinterpret_u16_f64", "arguments": [ - "poly64x1_t a" + "float64x1_t a" ], "return_type": { "value": "uint16x4_t" @@ -279670,7 +282381,6 @@ } }, "Architectures": [ - "A32", "A64" ], "instructions": [ @@ -279681,34 +282391,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_p8", - "arguments": [ - "poly8x8_t a" - ], - "return_type": { - "value": "uint16x4_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.8B" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_s16", + "name": "vreinterpret_u16_p16", "arguments": [ - "int16x4_t a" + "poly16x4_t a" ], "return_type": { "value": "uint16x4_t" @@ -279731,34 +282416,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_s32", - "arguments": [ - "int32x2_t a" - ], - "return_type": { - "value": "uint16x4_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.2S" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_s64", + "name": "vreinterpret_u16_p64", "arguments": [ - "int64x1_t a" + "poly64x1_t a" ], "return_type": { "value": "uint16x4_t" @@ -279769,7 +282429,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -279781,9 +282440,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_s8", + "name": "vreinterpret_u16_p8", "arguments": [ - "int8x8_t a" + "poly8x8_t a" ], "return_type": { "value": "uint16x4_t" @@ -279806,16 +282465,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_u32", + "name": "vreinterpret_u16_s16", "arguments": [ - "uint32x2_t a" + "int16x4_t a" ], "return_type": { "value": "uint16x4_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.2S" + "register": "Vd.4H" } }, "Architectures": [ @@ -279831,16 +282490,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_u64", + "name": "vreinterpret_u16_s32", "arguments": [ - "uint64x1_t a" + "int32x2_t a" ], "return_type": { "value": "uint16x4_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.1D" + "register": "Vd.2S" } }, "Architectures": [ @@ -279856,16 +282515,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u16_u8", + "name": "vreinterpret_u16_s64", "arguments": [ - "uint8x8_t a" + "int64x1_t a" ], "return_type": { "value": "uint16x4_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.8B" + "register": "Vd.1D" } }, "Architectures": [ @@ -279881,16 +282540,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_f16", + "name": "vreinterpret_u16_s8", "arguments": [ - "float16x4_t a" + "int8x8_t a" ], "return_type": { - "value": "uint32x2_t" + "value": "uint16x4_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4H" + "register": "Vd.8B" } }, "Architectures": [ @@ -279906,12 +282565,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_f32", + "name": "vreinterpret_u16_u32", "arguments": [ - "float32x2_t a" + "uint32x2_t a" ], "return_type": { - "value": "uint32x2_t" + "value": "uint16x4_t" }, "Arguments_Preparation": { "a": { @@ -279931,12 +282590,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_f64", + "name": "vreinterpret_u16_u64", "arguments": [ - "float64x1_t a" + "uint64x1_t a" ], "return_type": { - "value": "uint32x2_t" + "value": "uint16x4_t" }, "Arguments_Preparation": { "a": { @@ -279944,6 +282603,8 @@ } }, "Architectures": [ + "v7", + "A32", "A64" ], "instructions": [ @@ -279954,16 +282615,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_p16", + "name": "vreinterpret_u16_u8", "arguments": [ - "poly16x4_t a" + "uint8x8_t a" ], "return_type": { - "value": "uint32x2_t" + "value": "uint16x4_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4H" + "register": "Vd.8B" } }, "Architectures": [ @@ -279979,16 +282640,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_p64", + "name": "vreinterpret_u32_bf16", "arguments": [ - "poly64x1_t a" + "bfloat16x4_t a" ], "return_type": { "value": "uint32x2_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.1D" + "register": "Vd.4H" } }, "Architectures": [ @@ -280003,16 +282664,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_p8", + "name": "vreinterpret_u32_f16", "arguments": [ - "poly8x8_t a" + "float16x4_t a" ], "return_type": { "value": "uint32x2_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.8B" + "register": "Vd.4H" } }, "Architectures": [ @@ -280028,16 +282689,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_s16", + "name": "vreinterpret_u32_f32", "arguments": [ - "int16x4_t a" + "float32x2_t a" ], "return_type": { "value": "uint32x2_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4H" + "register": "Vd.2S" } }, "Architectures": [ @@ -280053,16 +282714,39 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_s32", + "name": "vreinterpret_u32_f64", "arguments": [ - "int32x2_t a" + "float64x1_t a" ], "return_type": { "value": "uint32x2_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.2S" + "register": "Vd.1D" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_u32_p16", + "arguments": [ + "poly16x4_t a" + ], + "return_type": { + "value": "uint32x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" } }, "Architectures": [ @@ -280078,9 +282762,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_s64", + "name": "vreinterpret_u32_p64", "arguments": [ - "int64x1_t a" + "poly64x1_t a" ], "return_type": { "value": "uint32x2_t" @@ -280091,7 +282775,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -280103,9 +282786,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_s8", + "name": "vreinterpret_u32_p8", "arguments": [ - "int8x8_t a" + "poly8x8_t a" ], "return_type": { "value": "uint32x2_t" @@ -280128,9 +282811,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_u16", + "name": "vreinterpret_u32_s16", "arguments": [ - "uint16x4_t a" + "int16x4_t a" ], "return_type": { "value": "uint32x2_t" @@ -280153,16 +282836,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_u64", + "name": "vreinterpret_u32_s32", "arguments": [ - "uint64x1_t a" + "int32x2_t a" ], "return_type": { "value": "uint32x2_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.1D" + "register": "Vd.2S" } }, "Architectures": [ @@ -280178,16 +282861,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u32_u8", + "name": "vreinterpret_u32_s64", "arguments": [ - "uint8x8_t a" + "int64x1_t a" ], "return_type": { "value": "uint32x2_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.8B" + "register": "Vd.1D" } }, "Architectures": [ @@ -280203,16 +282886,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_f16", + "name": "vreinterpret_u32_s8", "arguments": [ - "float16x4_t a" + "int8x8_t a" ], "return_type": { - "value": "uint64x1_t" + "value": "uint32x2_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4H" + "register": "Vd.8B" } }, "Architectures": [ @@ -280228,16 +282911,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_f32", + "name": "vreinterpret_u32_u16", "arguments": [ - "float32x2_t a" + "uint16x4_t a" ], "return_type": { - "value": "uint64x1_t" + "value": "uint32x2_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.2S" + "register": "Vd.4H" } }, "Architectures": [ @@ -280253,12 +282936,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_f64", + "name": "vreinterpret_u32_u64", "arguments": [ - "float64x1_t a" + "uint64x1_t a" ], "return_type": { - "value": "uint64x1_t" + "value": "uint32x2_t" }, "Arguments_Preparation": { "a": { @@ -280266,6 +282949,8 @@ } }, "Architectures": [ + "v7", + "A32", "A64" ], "instructions": [ @@ -280276,16 +282961,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_p16", + "name": "vreinterpret_u32_u8", "arguments": [ - "poly16x4_t a" + "uint8x8_t a" ], "return_type": { - "value": "uint64x1_t" + "value": "uint32x2_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4H" + "register": "Vd.8B" } }, "Architectures": [ @@ -280301,16 +282986,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_p64", + "name": "vreinterpret_u64_bf16", "arguments": [ - "poly64x1_t a" + "bfloat16x4_t a" ], "return_type": { "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.1D" + "register": "Vd.4H" } }, "Architectures": [ @@ -280325,16 +283010,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_p8", + "name": "vreinterpret_u64_f16", "arguments": [ - "poly8x8_t a" + "float16x4_t a" ], "return_type": { "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.8B" + "register": "Vd.4H" } }, "Architectures": [ @@ -280350,16 +283035,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_s16", + "name": "vreinterpret_u64_f32", "arguments": [ - "int16x4_t a" + "float32x2_t a" ], "return_type": { "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4H" + "register": "Vd.2S" } }, "Architectures": [ @@ -280375,21 +283060,19 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_s32", + "name": "vreinterpret_u64_f64", "arguments": [ - "int32x2_t a" + "float64x1_t a" ], "return_type": { "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.2S" + "register": "Vd.1D" } }, "Architectures": [ - "v7", - "A32", "A64" ], "instructions": [ @@ -280400,16 +283083,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_s64", + "name": "vreinterpret_u64_p16", "arguments": [ - "int64x1_t a" + "poly16x4_t a" ], "return_type": { "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.1D" + "register": "Vd.4H" } }, "Architectures": [ @@ -280425,20 +283108,19 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_s8", + "name": "vreinterpret_u64_p64", "arguments": [ - "int8x8_t a" + "poly64x1_t a" ], "return_type": { "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.8B" + "register": "Vd.1D" } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -280450,16 +283132,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_u16", + "name": "vreinterpret_u64_p8", "arguments": [ - "uint16x4_t a" + "poly8x8_t a" ], "return_type": { "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4H" + "register": "Vd.8B" } }, "Architectures": [ @@ -280475,16 +283157,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_u32", + "name": "vreinterpret_u64_s16", "arguments": [ - "uint32x2_t a" + "int16x4_t a" ], "return_type": { "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.2S" + "register": "Vd.4H" } }, "Architectures": [ @@ -280500,16 +283182,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u64_u8", + "name": "vreinterpret_u64_s32", "arguments": [ - "uint8x8_t a" + "int32x2_t a" ], "return_type": { "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.8B" + "register": "Vd.2S" } }, "Architectures": [ @@ -280525,16 +283207,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u8_f16", + "name": "vreinterpret_u64_s64", "arguments": [ - "float16x4_t a" + "int64x1_t a" ], "return_type": { - "value": "uint8x8_t" + "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4H" + "register": "Vd.1D" } }, "Architectures": [ @@ -280550,16 +283232,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u8_f32", + "name": "vreinterpret_u64_s8", "arguments": [ - "float32x2_t a" + "int8x8_t a" ], "return_type": { - "value": "uint8x8_t" + "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.2S" + "register": "Vd.8B" } }, "Architectures": [ @@ -280575,19 +283257,21 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u8_f64", + "name": "vreinterpret_u64_u16", "arguments": [ - "float64x1_t a" + "uint16x4_t a" ], "return_type": { - "value": "uint8x8_t" + "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.1D" + "register": "Vd.4H" } }, "Architectures": [ + "v7", + "A32", "A64" ], "instructions": [ @@ -280598,16 +283282,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u8_p16", + "name": "vreinterpret_u64_u32", "arguments": [ - "poly16x4_t a" + "uint32x2_t a" ], "return_type": { - "value": "uint8x8_t" + "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4H" + "register": "Vd.2S" } }, "Architectures": [ @@ -280623,19 +283307,20 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u8_p64", + "name": "vreinterpret_u64_u8", "arguments": [ - "poly64x1_t a" + "uint8x8_t a" ], "return_type": { - "value": "uint8x8_t" + "value": "uint64x1_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.1D" + "register": "Vd.8B" } }, "Architectures": [ + "v7", "A32", "A64" ], @@ -280647,20 +283332,19 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u8_p8", + "name": "vreinterpret_u8_bf16", "arguments": [ - "poly8x8_t a" + "bfloat16x4_t a" ], "return_type": { "value": "uint8x8_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.8B" + "register": "Vd.4H" } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -280672,9 +283356,156 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpret_u8_s16", + "name": "vreinterpret_u8_f16", "arguments": [ - "int16x4_t a" + "float16x4_t a" + ], + "return_type": { + "value": "uint8x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_u8_f32", + "arguments": [ + "float32x2_t a" + ], + "return_type": { + "value": "uint8x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2S" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_u8_f64", + "arguments": [ + "float64x1_t a" + ], + "return_type": { + "value": "uint8x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.1D" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_u8_p16", + "arguments": [ + "poly16x4_t a" + ], + "return_type": { + "value": "uint8x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_u8_p64", + "arguments": [ + "poly64x1_t a" + ], + "return_type": { + "value": "uint8x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.1D" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_u8_p8", + "arguments": [ + "poly8x8_t a" + ], + "return_type": { + "value": "uint8x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpret_u8_s16", + "arguments": [ + "int16x4_t a" ], "return_type": { "value": "uint8x8_t" @@ -280845,6 +283676,341 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_f32", + "arguments": [ + "float32x4_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_f64", + "arguments": [ + "float64x2_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2D" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_p128", + "arguments": [ + "poly128_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.1Q" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_p16", + "arguments": [ + "poly16x8_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_p64", + "arguments": [ + "poly64x2_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2D" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_p8", + "arguments": [ + "poly8x16_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.16B" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_s16", + "arguments": [ + "int16x8_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_s32", + "arguments": [ + "int32x4_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_s64", + "arguments": [ + "int64x2_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2D" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_s8", + "arguments": [ + "int8x16_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.16B" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_u16", + "arguments": [ + "uint16x8_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_u32", + "arguments": [ + "uint32x4_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_u64", + "arguments": [ + "uint64x2_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2D" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_bf16_u8", + "arguments": [ + "uint8x16_t a" + ], + "return_type": { + "value": "bfloat16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.16B" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_f16_f32", @@ -280993,9 +284159,109 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_f16_s16", + "name": "vreinterpretq_f16_s16", + "arguments": [ + "int16x8_t a" + ], + "return_type": { + "value": "float16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_f16_s32", + "arguments": [ + "int32x4_t a" + ], + "return_type": { + "value": "float16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_f16_s64", + "arguments": [ + "int64x2_t a" + ], + "return_type": { + "value": "float16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2D" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_f16_s8", + "arguments": [ + "int8x16_t a" + ], + "return_type": { + "value": "float16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.16B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_f16_u16", "arguments": [ - "int16x8_t a" + "uint16x8_t a" ], "return_type": { "value": "float16x8_t" @@ -281018,9 +284284,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_f16_s32", + "name": "vreinterpretq_f16_u32", "arguments": [ - "int32x4_t a" + "uint32x4_t a" ], "return_type": { "value": "float16x8_t" @@ -281043,9 +284309,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_f16_s64", + "name": "vreinterpretq_f16_u64", "arguments": [ - "int64x2_t a" + "uint64x2_t a" ], "return_type": { "value": "float16x8_t" @@ -281068,9 +284334,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_f16_s8", + "name": "vreinterpretq_f16_u8", "arguments": [ - "int8x16_t a" + "uint8x16_t a" ], "return_type": { "value": "float16x8_t" @@ -281093,12 +284359,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_f16_u16", + "name": "vreinterpretq_f32_bf16", "arguments": [ - "uint16x8_t a" + "bfloat16x8_t a" ], "return_type": { - "value": "float16x8_t" + "value": "float32x4_t" }, "Arguments_Preparation": { "a": { @@ -281106,82 +284372,6 @@ } }, "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_f16_u32", - "arguments": [ - "uint32x4_t a" - ], - "return_type": { - "value": "float16x8_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.4S" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_f16_u64", - "arguments": [ - "uint64x2_t a" - ], - "return_type": { - "value": "float16x8_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.2D" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_f16_u8", - "arguments": [ - "uint8x16_t a" - ], - "return_type": { - "value": "float16x8_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.16B" - } - }, - "Architectures": [ - "v7", "A32", "A64" ], @@ -281489,6 +284679,29 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_f64_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "float64x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_f64_f16", @@ -281813,6 +285026,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_p128_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "poly128_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_p128_f16", @@ -281863,9 +285100,128 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_f64", + "name": "vreinterpretq_p128_f64", + "arguments": [ + "float64x2_t a" + ], + "return_type": { + "value": "poly128_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.1Q" + } + }, + "Architectures": [ + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_p128_p16", + "arguments": [ + "poly16x8_t a" + ], + "return_type": { + "value": "poly128_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_p128_p8", + "arguments": [ + "poly8x16_t a" + ], + "return_type": { + "value": "poly128_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.16B" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_p128_s16", + "arguments": [ + "int16x8_t a" + ], + "return_type": { + "value": "poly128_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_p128_s32", + "arguments": [ + "int32x4_t a" + ], + "return_type": { + "value": "poly128_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_p128_s64", "arguments": [ - "float64x2_t a" + "int64x2_t a" ], "return_type": { "value": "poly128_t" @@ -281875,29 +285231,6 @@ "register": "Vd.1Q" } }, - "Architectures": [ - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_p16", - "arguments": [ - "poly16x8_t a" - ], - "return_type": { - "value": "poly128_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.8H" - } - }, "Architectures": [ "A32", "A64" @@ -281910,9 +285243,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_p8", + "name": "vreinterpretq_p128_s8", "arguments": [ - "poly8x16_t a" + "int8x16_t a" ], "return_type": { "value": "poly128_t" @@ -281934,9 +285267,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_s16", + "name": "vreinterpretq_p128_u16", "arguments": [ - "int16x8_t a" + "uint16x8_t a" ], "return_type": { "value": "poly128_t" @@ -281958,9 +285291,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_s32", + "name": "vreinterpretq_p128_u32", "arguments": [ - "int32x4_t a" + "uint32x4_t a" ], "return_type": { "value": "poly128_t" @@ -281982,9 +285315,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_s64", + "name": "vreinterpretq_p128_u64", "arguments": [ - "int64x2_t a" + "uint64x2_t a" ], "return_type": { "value": "poly128_t" @@ -282006,9 +285339,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_s8", + "name": "vreinterpretq_p128_u8", "arguments": [ - "int8x16_t a" + "uint8x16_t a" ], "return_type": { "value": "poly128_t" @@ -282030,12 +285363,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_u16", + "name": "vreinterpretq_p16_bf16", "arguments": [ - "uint16x8_t a" + "bfloat16x8_t a" ], "return_type": { - "value": "poly128_t" + "value": "poly16x8_t" }, "Arguments_Preparation": { "a": { @@ -282052,78 +285385,6 @@ ] ] }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_u32", - "arguments": [ - "uint32x4_t a" - ], - "return_type": { - "value": "poly128_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.4S" - } - }, - "Architectures": [ - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_u64", - "arguments": [ - "uint64x2_t a" - ], - "return_type": { - "value": "poly128_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.1Q" - } - }, - "Architectures": [ - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_p128_u8", - "arguments": [ - "uint8x16_t a" - ], - "return_type": { - "value": "poly128_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.16B" - } - }, - "Architectures": [ - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_p16_f16", @@ -282470,6 +285731,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_p64_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "poly64x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_p64_f16", @@ -282781,6 +286066,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_p8_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "poly8x16_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_p8_f16", @@ -283127,6 +286436,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_s16_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "int16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_s16_f16", @@ -283473,6 +286806,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_s32_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "int32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_s32_f16", @@ -283646,9 +287003,84 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_s32_s16", + "name": "vreinterpretq_s32_s16", + "arguments": [ + "int16x8_t a" + ], + "return_type": { + "value": "int32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_s32_s64", + "arguments": [ + "int64x2_t a" + ], + "return_type": { + "value": "int32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2D" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_s32_s8", + "arguments": [ + "int8x16_t a" + ], + "return_type": { + "value": "int32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.16B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_s32_u16", "arguments": [ - "int16x8_t a" + "uint16x8_t a" ], "return_type": { "value": "int32x4_t" @@ -283671,66 +287103,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_s32_s64", - "arguments": [ - "int64x2_t a" - ], - "return_type": { - "value": "int32x4_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.2D" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_s32_s8", - "arguments": [ - "int8x16_t a" - ], - "return_type": { - "value": "int32x4_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.16B" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_s32_u16", + "name": "vreinterpretq_s32_u32", "arguments": [ - "uint16x8_t a" + "uint32x4_t a" ], "return_type": { "value": "int32x4_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.8H" + "register": "Vd.4S" } }, "Architectures": [ @@ -283746,16 +287128,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_s32_u32", + "name": "vreinterpretq_s32_u64", "arguments": [ - "uint32x4_t a" + "uint64x2_t a" ], "return_type": { "value": "int32x4_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.4S" + "register": "Vd.2D" } }, "Architectures": [ @@ -283771,16 +287153,16 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_s32_u64", + "name": "vreinterpretq_s32_u8", "arguments": [ - "uint64x2_t a" + "uint8x16_t a" ], "return_type": { "value": "int32x4_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.2D" + "register": "Vd.16B" } }, "Architectures": [ @@ -283796,20 +287178,19 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_s32_u8", + "name": "vreinterpretq_s64_bf16", "arguments": [ - "uint8x16_t a" + "bfloat16x8_t a" ], "return_type": { - "value": "int32x4_t" + "value": "int64x2_t" }, "Arguments_Preparation": { "a": { - "register": "Vd.16B" + "register": "Vd.8H" } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -284165,6 +287546,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_s8_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "int8x16_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_s8_f16", @@ -284511,6 +287916,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_u16_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "uint16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_u16_f16", @@ -284857,6 +288286,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_u32_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "uint32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_u32_f16", @@ -285203,6 +288656,30 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_u64_bf16", + "arguments": [ + "bfloat16x8_t a" + ], + "return_type": { + "value": "uint64x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vreinterpretq_u64_f16", @@ -285376,9 +288853,109 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_u64_s16", + "name": "vreinterpretq_u64_s16", + "arguments": [ + "int16x8_t a" + ], + "return_type": { + "value": "uint64x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_u64_s32", + "arguments": [ + "int32x4_t a" + ], + "return_type": { + "value": "uint64x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4S" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_u64_s64", + "arguments": [ + "int64x2_t a" + ], + "return_type": { + "value": "uint64x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2D" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_u64_s8", + "arguments": [ + "int8x16_t a" + ], + "return_type": { + "value": "uint64x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.16B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "NOP" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vreinterpretq_u64_u16", "arguments": [ - "int16x8_t a" + "uint16x8_t a" ], "return_type": { "value": "uint64x2_t" @@ -285401,9 +288978,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_u64_s32", + "name": "vreinterpretq_u64_u32", "arguments": [ - "int32x4_t a" + "uint32x4_t a" ], "return_type": { "value": "uint64x2_t" @@ -285426,34 +289003,9 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_u64_s64", - "arguments": [ - "int64x2_t a" - ], - "return_type": { - "value": "uint64x2_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.2D" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_u64_s8", + "name": "vreinterpretq_u64_u8", "arguments": [ - "int8x16_t a" + "uint8x16_t a" ], "return_type": { "value": "uint64x2_t" @@ -285476,12 +289028,12 @@ }, { "SIMD_ISA": "Neon", - "name": "vreinterpretq_u64_u16", + "name": "vreinterpretq_u8_bf16", "arguments": [ - "uint16x8_t a" + "bfloat16x8_t a" ], "return_type": { - "value": "uint64x2_t" + "value": "uint8x16_t" }, "Arguments_Preparation": { "a": { @@ -285489,57 +289041,6 @@ } }, "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_u64_u32", - "arguments": [ - "uint32x4_t a" - ], - "return_type": { - "value": "uint64x2_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.4S" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "NOP" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vreinterpretq_u64_u8", - "arguments": [ - "uint8x16_t a" - ], - "return_type": { - "value": "uint64x2_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Vd.16B" - } - }, - "Architectures": [ - "v7", "A32", "A64" ], @@ -291788,6 +295289,39 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vset_lane_bf16", + "arguments": [ + "bfloat16_t a", + "bfloat16x4_t v", + "const int lane" + ], + "return_type": { + "value": "bfloat16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "VnH" + }, + "lane": { + "minimum": 0, + "maximum": 3 + }, + "v": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "INS" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vset_lane_f16", @@ -292093,14 +295627,150 @@ }, { "SIMD_ISA": "Neon", - "name": "vset_lane_s8", + "name": "vset_lane_s8", + "arguments": [ + "int8_t a", + "int8x8_t v", + "const int lane" + ], + "return_type": { + "value": "int8x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Rn" + }, + "lane": { + "minimum": 0, + "maximum": 7 + }, + "v": { + "register": "Vd.8B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "MOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vset_lane_u16", + "arguments": [ + "uint16_t a", + "uint16x4_t v", + "const int lane" + ], + "return_type": { + "value": "uint16x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Rn" + }, + "lane": { + "minimum": 0, + "maximum": 3 + }, + "v": { + "register": "Vd.4H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "MOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vset_lane_u32", + "arguments": [ + "uint32_t a", + "uint32x2_t v", + "const int lane" + ], + "return_type": { + "value": "uint32x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Rn" + }, + "lane": { + "minimum": 0, + "maximum": 1 + }, + "v": { + "register": "Vd.2S" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "MOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vset_lane_u64", + "arguments": [ + "uint64_t a", + "uint64x1_t v", + "const int lane" + ], + "return_type": { + "value": "uint64x1_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Rn" + }, + "lane": { + "minimum": 0, + "maximum": 0 + }, + "v": { + "register": "Vd.1D" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "MOV" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vset_lane_u8", "arguments": [ - "int8_t a", - "int8x8_t v", + "uint8_t a", + "uint8x8_t v", "const int lane" ], "return_type": { - "value": "int8x8_t" + "value": "uint8x8_t" }, "Arguments_Preparation": { "a": { @@ -292127,137 +295797,34 @@ }, { "SIMD_ISA": "Neon", - "name": "vset_lane_u16", + "name": "vsetq_lane_bf16", "arguments": [ - "uint16_t a", - "uint16x4_t v", + "bfloat16_t a", + "bfloat16x8_t v", "const int lane" ], "return_type": { - "value": "uint16x4_t" + "value": "bfloat16x8_t" }, "Arguments_Preparation": { "a": { - "register": "Rn" - }, - "lane": { - "minimum": 0, - "maximum": 3 - }, - "v": { - "register": "Vd.4H" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "MOV" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vset_lane_u32", - "arguments": [ - "uint32_t a", - "uint32x2_t v", - "const int lane" - ], - "return_type": { - "value": "uint32x2_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Rn" - }, - "lane": { - "minimum": 0, - "maximum": 1 - }, - "v": { - "register": "Vd.2S" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "MOV" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vset_lane_u64", - "arguments": [ - "uint64_t a", - "uint64x1_t v", - "const int lane" - ], - "return_type": { - "value": "uint64x1_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Rn" - }, - "lane": { - "minimum": 0, - "maximum": 0 - }, - "v": { - "register": "Vd.1D" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "MOV" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vset_lane_u8", - "arguments": [ - "uint8_t a", - "uint8x8_t v", - "const int lane" - ], - "return_type": { - "value": "uint8x8_t" - }, - "Arguments_Preparation": { - "a": { - "register": "Rn" + "register": "VnH" }, "lane": { "minimum": 0, "maximum": 7 }, "v": { - "register": "Vd.8B" + "register": "Vd.8H" } }, "Architectures": [ - "v7", "A32", "A64" ], "instructions": [ [ - "MOV" + "INS" ] ] }, @@ -298339,137 +301906,267 @@ }, { "SIMD_ISA": "Neon", - "name": "vsriq_n_u16", + "name": "vsriq_n_u16", + "arguments": [ + "uint16x8_t a", + "uint16x8_t b", + "const int n" + ], + "return_type": { + "value": "uint16x8_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.8H" + }, + "b": { + "register": "Vn.8H" + }, + "n": { + "minimum": 1, + "maximum": 16 + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "SRI" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vsriq_n_u32", + "arguments": [ + "uint32x4_t a", + "uint32x4_t b", + "const int n" + ], + "return_type": { + "value": "uint32x4_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.4S" + }, + "b": { + "register": "Vn.4S" + }, + "n": { + "minimum": 1, + "maximum": 32 + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "SRI" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vsriq_n_u64", + "arguments": [ + "uint64x2_t a", + "uint64x2_t b", + "const int n" + ], + "return_type": { + "value": "uint64x2_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.2D" + }, + "b": { + "register": "Vn.2D" + }, + "n": { + "minimum": 1, + "maximum": 64 + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "SRI" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vsriq_n_u8", + "arguments": [ + "uint8x16_t a", + "uint8x16_t b", + "const int n" + ], + "return_type": { + "value": "uint8x16_t" + }, + "Arguments_Preparation": { + "a": { + "register": "Vd.16B" + }, + "b": { + "register": "Vn.16B" + }, + "n": { + "minimum": 1, + "maximum": 8 + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "SRI" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vst1_bf16", "arguments": [ - "uint16x8_t a", - "uint16x8_t b", - "const int n" + "bfloat16_t * ptr", + "bfloat16x4_t val" ], "return_type": { - "value": "uint16x8_t" + "value": "void" }, "Arguments_Preparation": { - "a": { - "register": "Vd.8H" - }, - "b": { - "register": "Vn.8H" + "ptr": { + "register": "Xn" }, - "n": { - "minimum": 1, - "maximum": 16 + "val": { + "register": "Vt.4H" } }, "Architectures": [ - "v7", "A32", "A64" ], "instructions": [ [ - "SRI" + "ST1" ] ] }, { "SIMD_ISA": "Neon", - "name": "vsriq_n_u32", + "name": "vst1_bf16_x2", "arguments": [ - "uint32x4_t a", - "uint32x4_t b", - "const int n" + "bfloat16_t * ptr", + "bfloat16x4x2_t val" ], "return_type": { - "value": "uint32x4_t" + "value": "void" }, "Arguments_Preparation": { - "a": { - "register": "Vd.4S" + "ptr": { + "register": "Xn" }, - "b": { - "register": "Vn.4S" + "val.val[0]": { + "register": "Vt.4H" }, - "n": { - "minimum": 1, - "maximum": 32 + "val.val[1]": { + "register": "Vt2.4H" } }, "Architectures": [ - "v7", "A32", "A64" ], "instructions": [ [ - "SRI" + "ST1" ] ] }, { "SIMD_ISA": "Neon", - "name": "vsriq_n_u64", + "name": "vst1_bf16_x3", "arguments": [ - "uint64x2_t a", - "uint64x2_t b", - "const int n" + "bfloat16_t * ptr", + "bfloat16x4x3_t val" ], "return_type": { - "value": "uint64x2_t" + "value": "void" }, "Arguments_Preparation": { - "a": { - "register": "Vd.2D" + "ptr": { + "register": "Xn" }, - "b": { - "register": "Vn.2D" + "val.val[0]": { + "register": "Vt.4H" }, - "n": { - "minimum": 1, - "maximum": 64 + "val.val[1]": { + "register": "Vt2.4H" + }, + "val.val[2]": { + "register": "Vt3.4H" } }, "Architectures": [ - "v7", "A32", "A64" ], "instructions": [ [ - "SRI" + "ST1" ] ] }, { "SIMD_ISA": "Neon", - "name": "vsriq_n_u8", + "name": "vst1_bf16_x4", "arguments": [ - "uint8x16_t a", - "uint8x16_t b", - "const int n" + "bfloat16_t * ptr", + "bfloat16x4x4_t val" ], "return_type": { - "value": "uint8x16_t" + "value": "void" }, "Arguments_Preparation": { - "a": { - "register": "Vd.16B" + "ptr": { + "register": "Xn" }, - "b": { - "register": "Vn.16B" + "val.val[0]": { + "register": "Vt.4H" }, - "n": { - "minimum": 1, - "maximum": 8 + "val.val[1]": { + "register": "Vt2.4H" + }, + "val.val[2]": { + "register": "Vt3.4H" + }, + "val.val[3]": { + "register": "Vt4.4H" } }, "Architectures": [ - "v7", "A32", "A64" ], "instructions": [ [ - "SRI" + "ST1" ] ] }, @@ -298867,6 +302564,39 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst1_lane_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x4_t val", + "const int lane" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "ptr": { + "register": "Xn" + }, + "val": { + "register": "Vt.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST1" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst1_lane_f16", @@ -300846,6 +304576,136 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst1q_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x8_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val": { + "register": "Vt.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vst1q_bf16_x2", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x8x2_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.8H" + }, + "val.val[1]": { + "register": "Vt2.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vst1q_bf16_x3", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x8x3_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.8H" + }, + "val.val[1]": { + "register": "Vt2.8H" + }, + "val.val[2]": { + "register": "Vt3.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vst1q_bf16_x4", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x8x4_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.8H" + }, + "val.val[1]": { + "register": "Vt2.8H" + }, + "val.val[2]": { + "register": "Vt3.8H" + }, + "val.val[3]": { + "register": "Vt4.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST1" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst1q_f16", @@ -301240,6 +305100,39 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst1q_lane_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x8_t val", + "const int lane" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "ptr": { + "register": "Xn" + }, + "val": { + "register": "Vt.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST1" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst1q_lane_f16", @@ -303220,6 +307113,37 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst2_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x4x2_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.4H" + }, + "val.val[1]": { + "register": "Vt2.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST2" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst2_f16", @@ -303314,6 +307238,42 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst2_lane_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x4x2_t val", + "const int lane" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.4H" + }, + "val.val[1]": { + "register": "Vt2.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST2" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst2_lane_f16", @@ -304175,6 +308135,37 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst2q_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x8x2_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.8H" + }, + "val.val[1]": { + "register": "Vt2.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST2" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst2q_f16", @@ -304269,6 +308260,42 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst2q_lane_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x8x2_t val", + "const int lane" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.8H" + }, + "val.val[1]": { + "register": "Vt2.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST2" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst2q_lane_f16", @@ -305119,6 +309146,40 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst3_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x4x3_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.4H" + }, + "val.val[1]": { + "register": "Vt2.4H" + }, + "val.val[2]": { + "register": "Vt3.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST3" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst3_f16", @@ -305222,6 +309283,45 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst3_lane_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x4x3_t val", + "const int lane" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.4H" + }, + "val.val[1]": { + "register": "Vt2.4H" + }, + "val.val[2]": { + "register": "Vt3.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST3" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst3_lane_f16", @@ -306020,10 +310120,150 @@ }, { "SIMD_ISA": "Neon", - "name": "vst3_u16", + "name": "vst3_u16", + "arguments": [ + "uint16_t * ptr", + "uint16x4x3_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.4H" + }, + "val.val[1]": { + "register": "Vt2.4H" + }, + "val.val[2]": { + "register": "Vt3.4H" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "ST3" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vst3_u32", + "arguments": [ + "uint32_t * ptr", + "uint32x2x3_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.2S" + }, + "val.val[1]": { + "register": "Vt2.2S" + }, + "val.val[2]": { + "register": "Vt3.2S" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "ST3" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vst3_u64", + "arguments": [ + "uint64_t * ptr", + "uint64x1x3_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.1D" + }, + "val.val[1]": { + "register": "Vt2.1D" + }, + "val.val[2]": { + "register": "Vt3.1D" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "ST1" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vst3_u8", + "arguments": [ + "uint8_t * ptr", + "uint8x8x3_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.8B" + }, + "val.val[1]": { + "register": "Vt2.8B" + }, + "val.val[2]": { + "register": "Vt3.8B" + } + }, + "Architectures": [ + "v7", + "A32", + "A64" + ], + "instructions": [ + [ + "ST3" + ] + ] + }, + { + "SIMD_ISA": "Neon", + "name": "vst3q_bf16", "arguments": [ - "uint16_t * ptr", - "uint16x4x3_t val" + "bfloat16_t * ptr", + "bfloat16x8x3_t val" ], "return_type": { "value": "void" @@ -306033,17 +310273,16 @@ "register": "Xn" }, "val.val[0]": { - "register": "Vt.4H" + "register": "Vt.8H" }, "val.val[1]": { - "register": "Vt2.4H" + "register": "Vt2.8H" }, "val.val[2]": { - "register": "Vt3.4H" + "register": "Vt3.8H" } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -306055,10 +310294,10 @@ }, { "SIMD_ISA": "Neon", - "name": "vst3_u32", + "name": "vst3q_f16", "arguments": [ - "uint32_t * ptr", - "uint32x2x3_t val" + "float16_t * ptr", + "float16x8x3_t val" ], "return_type": { "value": "void" @@ -306068,13 +310307,13 @@ "register": "Xn" }, "val.val[0]": { - "register": "Vt.2S" + "register": "Vt.8H" }, "val.val[1]": { - "register": "Vt2.2S" + "register": "Vt2.8H" }, "val.val[2]": { - "register": "Vt3.2S" + "register": "Vt3.8H" } }, "Architectures": [ @@ -306090,10 +310329,10 @@ }, { "SIMD_ISA": "Neon", - "name": "vst3_u64", + "name": "vst3q_f32", "arguments": [ - "uint64_t * ptr", - "uint64x1x3_t val" + "float32_t * ptr", + "float32x4x3_t val" ], "return_type": { "value": "void" @@ -306103,13 +310342,13 @@ "register": "Xn" }, "val.val[0]": { - "register": "Vt.1D" + "register": "Vt.4S" }, "val.val[1]": { - "register": "Vt2.1D" + "register": "Vt2.4S" }, "val.val[2]": { - "register": "Vt3.1D" + "register": "Vt3.4S" } }, "Architectures": [ @@ -306119,16 +310358,16 @@ ], "instructions": [ [ - "ST1" + "ST3" ] ] }, { "SIMD_ISA": "Neon", - "name": "vst3_u8", + "name": "vst3q_f64", "arguments": [ - "uint8_t * ptr", - "uint8x8x3_t val" + "float64_t * ptr", + "float64x2x3_t val" ], "return_type": { "value": "void" @@ -306138,18 +310377,16 @@ "register": "Xn" }, "val.val[0]": { - "register": "Vt.8B" + "register": "Vt.2D" }, "val.val[1]": { - "register": "Vt2.8B" + "register": "Vt2.2D" }, "val.val[2]": { - "register": "Vt3.8B" + "register": "Vt3.2D" } }, "Architectures": [ - "v7", - "A32", "A64" ], "instructions": [ @@ -306160,15 +310397,20 @@ }, { "SIMD_ISA": "Neon", - "name": "vst3q_f16", + "name": "vst3q_lane_bf16", "arguments": [ - "float16_t * ptr", - "float16x8x3_t val" + "bfloat16_t * ptr", + "bfloat16x8x3_t val", + "const int lane" ], "return_type": { "value": "void" }, "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, "ptr": { "register": "Xn" }, @@ -306183,7 +310425,6 @@ } }, "Architectures": [ - "v7", "A32", "A64" ], @@ -306193,74 +310434,6 @@ ] ] }, - { - "SIMD_ISA": "Neon", - "name": "vst3q_f32", - "arguments": [ - "float32_t * ptr", - "float32x4x3_t val" - ], - "return_type": { - "value": "void" - }, - "Arguments_Preparation": { - "ptr": { - "register": "Xn" - }, - "val.val[0]": { - "register": "Vt.4S" - }, - "val.val[1]": { - "register": "Vt2.4S" - }, - "val.val[2]": { - "register": "Vt3.4S" - } - }, - "Architectures": [ - "v7", - "A32", - "A64" - ], - "instructions": [ - [ - "ST3" - ] - ] - }, - { - "SIMD_ISA": "Neon", - "name": "vst3q_f64", - "arguments": [ - "float64_t * ptr", - "float64x2x3_t val" - ], - "return_type": { - "value": "void" - }, - "Arguments_Preparation": { - "ptr": { - "register": "Xn" - }, - "val.val[0]": { - "register": "Vt.2D" - }, - "val.val[1]": { - "register": "Vt2.2D" - }, - "val.val[2]": { - "register": "Vt3.2D" - } - }, - "Architectures": [ - "A64" - ], - "instructions": [ - [ - "ST3" - ] - ] - }, { "SIMD_ISA": "Neon", "name": "vst3q_lane_f16", @@ -307192,6 +311365,43 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst4_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x4x4_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.4H" + }, + "val.val[1]": { + "register": "Vt2.4H" + }, + "val.val[2]": { + "register": "Vt3.4H" + }, + "val.val[3]": { + "register": "Vt4.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST4" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst4_f16", @@ -307304,6 +311514,48 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst4_lane_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x4x4_t val", + "const int lane" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 3 + }, + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.4H" + }, + "val.val[1]": { + "register": "Vt2.4H" + }, + "val.val[2]": { + "register": "Vt3.4H" + }, + "val.val[3]": { + "register": "Vt4.4H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST4" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst4_lane_f16", @@ -308315,6 +312567,43 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst4q_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x8x4_t val" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.8H" + }, + "val.val[1]": { + "register": "Vt2.8H" + }, + "val.val[2]": { + "register": "Vt3.8H" + }, + "val.val[3]": { + "register": "Vt4.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST4" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst4q_f16", @@ -308427,6 +312716,48 @@ ] ] }, + { + "SIMD_ISA": "Neon", + "name": "vst4q_lane_bf16", + "arguments": [ + "bfloat16_t * ptr", + "bfloat16x8x4_t val", + "const int lane" + ], + "return_type": { + "value": "void" + }, + "Arguments_Preparation": { + "lane": { + "minimum": 0, + "maximum": 7 + }, + "ptr": { + "register": "Xn" + }, + "val.val[0]": { + "register": "Vt.8H" + }, + "val.val[1]": { + "register": "Vt2.8H" + }, + "val.val[2]": { + "register": "Vt3.8H" + }, + "val.val[3]": { + "register": "Vt4.8H" + } + }, + "Architectures": [ + "A32", + "A64" + ], + "instructions": [ + [ + "ST4" + ] + ] + }, { "SIMD_ISA": "Neon", "name": "vst4q_lane_f16", From 892a26de04187b5eb8ccd32bf6a2c4dc14de6624 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Mon, 26 Oct 2026 13:43:50 +0000 Subject: [PATCH 5/5] add tracking issue number --- .../core_arch/src/aarch64/neon/generated.rs | 28 +- .../src/arm_shared/neon/generated.rs | 344 +++++++++--------- .../spec/neon/aarch64.spec.yml | 4 +- .../spec/neon/arm_shared.spec.yml | 4 +- 4 files changed, 190 insertions(+), 190 deletions(-) diff --git a/crates/core_arch/src/aarch64/neon/generated.rs b/crates/core_arch/src/aarch64/neon/generated.rs index 3f2d55a9fe..72f65d0d14 100644 --- a/crates/core_arch/src/aarch64/neon/generated.rs +++ b/crates/core_arch/src/aarch64/neon/generated.rs @@ -4611,7 +4611,7 @@ pub fn vcombine_f64(a: float64x1_t, b: float64x1_t) -> float64x2_t { #[rustc_legacy_const_generics(1, 3)] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] pub fn vcopy_lane_bf16( a: bfloat16x4_t, b: bfloat16x4_t, @@ -4626,7 +4626,7 @@ pub fn vcopy_lane_bf16( #[rustc_legacy_const_generics(1, 3)] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] pub fn vcopyq_lane_bf16( a: bfloat16x8_t, b: bfloat16x4_t, @@ -4848,7 +4848,7 @@ pub fn vcopy_lane_p64( #[rustc_legacy_const_generics(1, 3)] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] pub fn vcopy_laneq_bf16( a: bfloat16x4_t, b: bfloat16x8_t, @@ -4863,7 +4863,7 @@ pub fn vcopy_laneq_bf16( #[rustc_legacy_const_generics(1, 3)] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] pub fn vcopyq_laneq_bf16( a: bfloat16x8_t, b: bfloat16x8_t, @@ -11480,7 +11480,7 @@ pub unsafe fn vldap1q_lane_p64(ptr: *const p64, src: poly64x2_t #[rustc_legacy_const_generics(2)] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] pub unsafe fn vluti2_lane_bf16(a: bfloat16x4_t, b: uint8x8_t) -> bfloat16x8_t { static_assert!(LANE >= 0 && LANE <= 3); unsafe extern "llvm-intrinsic" { @@ -11502,7 +11502,7 @@ pub unsafe fn vluti2_lane_bf16(a: bfloat16x4_t, b: uint8x8_t) - #[rustc_legacy_const_generics(2)] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] pub unsafe fn vluti2q_lane_bf16(a: bfloat16x8_t, b: uint8x8_t) -> bfloat16x8_t { static_assert!(LANE >= 0 && LANE <= 3); unsafe extern "llvm-intrinsic" { @@ -11734,7 +11734,7 @@ pub unsafe fn vluti2q_lane_s16(a: int16x8_t, b: uint8x8_t) -> i #[rustc_legacy_const_generics(2)] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] pub unsafe fn vluti2_laneq_bf16(a: bfloat16x4_t, b: uint8x16_t) -> bfloat16x8_t { static_assert!(LANE >= 0 && LANE <= 7); unsafe extern "llvm-intrinsic" { @@ -11756,7 +11756,7 @@ pub unsafe fn vluti2_laneq_bf16(a: bfloat16x4_t, b: uint8x16_t) #[rustc_legacy_const_generics(2)] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] pub unsafe fn vluti2q_laneq_bf16(a: bfloat16x8_t, b: uint8x16_t) -> bfloat16x8_t { static_assert!(LANE >= 0 && LANE <= 7); unsafe extern "llvm-intrinsic" { @@ -11988,7 +11988,7 @@ pub unsafe fn vluti2q_laneq_s16(a: int16x8_t, b: uint8x16_t) - #[rustc_legacy_const_generics(2)] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] pub unsafe fn vluti4q_lane_bf16_x2( a: bfloat16x8x2_t, b: uint8x8_t, @@ -12123,7 +12123,7 @@ pub unsafe fn vluti4q_lane_p8(a: poly8x16_t, b: uint8x8_t) -> p #[rustc_legacy_const_generics(2)] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] pub unsafe fn vluti4q_laneq_bf16_x2( a: bfloat16x8x2_t, b: uint8x16_t, @@ -21400,7 +21400,7 @@ pub fn vrecpxh_f16(a: f16) -> f16 { #[target_feature(enable = "neon")] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] #[cfg(not(target_arch = "arm64ec"))] #[cfg_attr(test, assert_instr(nop))] pub fn vreinterpret_bf16_f64(a: float64x1_t) -> bfloat16x4_t { @@ -21412,7 +21412,7 @@ pub fn vreinterpret_bf16_f64(a: float64x1_t) -> bfloat16x4_t { #[target_feature(enable = "neon")] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] #[cfg(not(target_arch = "arm64ec"))] #[cfg_attr(test, assert_instr(nop))] pub fn vreinterpretq_bf16_f64(a: float64x2_t) -> bfloat16x8_t { @@ -21424,7 +21424,7 @@ pub fn vreinterpretq_bf16_f64(a: float64x2_t) -> bfloat16x8_t { #[target_feature(enable = "neon")] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] #[cfg(not(target_arch = "arm64ec"))] #[cfg_attr(test, assert_instr(nop))] pub fn vreinterpret_f64_bf16(a: bfloat16x4_t) -> float64x1_t { @@ -21436,7 +21436,7 @@ pub fn vreinterpret_f64_bf16(a: bfloat16x4_t) -> float64x1_t { #[target_feature(enable = "neon")] #[cfg(any(target_has_reliable_f16b, doc))] #[doc(cfg(target_has_reliable_f16b))] -#[unstable(feature = "stdarch_neon_bf16", issue = "none")] +#[unstable(feature = "stdarch_neon_bf16", issue = "164986")] #[cfg(not(target_arch = "arm64ec"))] #[cfg_attr(test, assert_instr(nop))] pub fn vreinterpretq_f64_bf16(a: bfloat16x8_t) -> float64x2_t { diff --git a/crates/core_arch/src/arm_shared/neon/generated.rs b/crates/core_arch/src/arm_shared/neon/generated.rs index 684f4c5a79..15bc14228c 100644 --- a/crates/core_arch/src/arm_shared/neon/generated.rs +++ b/crates/core_arch/src/arm_shared/neon/generated.rs @@ -3762,7 +3762,7 @@ pub fn vandq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -3797,7 +3797,7 @@ pub fn vbfdot_f32(a: float32x2_t, b: bfloat16x4_t, c: bfloat16x4_t) -> float32x2 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -3832,7 +3832,7 @@ pub fn vbfdotq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32x #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -3868,7 +3868,7 @@ pub fn vbfdot_lane_f32( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -3904,7 +3904,7 @@ pub fn vbfdotq_lane_f32( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -3940,7 +3940,7 @@ pub fn vbfdot_laneq_f32( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -3977,7 +3977,7 @@ pub fn vbfdotq_laneq_f32( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -4013,7 +4013,7 @@ pub fn vbfmlalbq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float3 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -4054,7 +4054,7 @@ pub fn vbfmlalbq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float3 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -4089,7 +4089,7 @@ pub fn vbfmlalbq_lane_f32( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -4125,7 +4125,7 @@ pub fn vbfmlalbq_laneq_f32( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -4161,7 +4161,7 @@ pub fn vbfmlaltq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float3 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -4202,7 +4202,7 @@ pub fn vbfmlaltq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float3 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -4237,7 +4237,7 @@ pub fn vbfmlaltq_lane_f32( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -4273,7 +4273,7 @@ pub fn vbfmlaltq_laneq_f32( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -4309,7 +4309,7 @@ pub fn vbfmmlaq_f32(a: float32x4_t, b: bfloat16x8_t, c: bfloat16x8_t) -> float32 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -8355,7 +8355,7 @@ pub fn vcntq_p8(a: poly8x16_t) -> poly8x16_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -8379,7 +8379,7 @@ pub fn vcombine_bf16(a: bfloat16x4_t, b: bfloat16x4_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -8955,7 +8955,7 @@ pub fn vcombine_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -9253,7 +9253,7 @@ pub fn vcreate_p64(a: u64) -> poly64x1_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -9276,7 +9276,7 @@ pub fn vcvt_bf16_f32(a: float32x4_t) -> bfloat16x4_t { #[target_feature(enable = "neon,bf16")] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr( @@ -9406,7 +9406,7 @@ pub fn vcvtq_f16_u16(a: uint16x8_t) -> float16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10327,7 +10327,7 @@ pub fn vcvtq_u32_f32(a: float32x4_t) -> uint32x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10350,7 +10350,7 @@ pub fn vcvtah_f32_bf16(a: bfloat16_t) -> f32 { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10387,7 +10387,7 @@ pub fn vcvth_bf16_f32(a: f32) -> bfloat16_t { #[target_feature(enable = "neon,bf16")] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10427,7 +10427,7 @@ pub unsafe fn vcvtq_high_bf16_f32(_inactive: bfloat16x8_t, a: float32x4_t) -> bf #[target_feature(enable = "neon,bf16")] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10470,7 +10470,7 @@ pub unsafe fn vcvtq_high_bf16_f32(_inactive: bfloat16x8_t, a: float32x4_t) -> bf #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10496,7 +10496,7 @@ pub unsafe fn vcvtq_high_bf16_f32(_inactive: bfloat16x8_t, a: float32x4_t) -> bf #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10533,7 +10533,7 @@ pub fn vcvtq_high_f32_bf16(a: bfloat16x8_t) -> float32x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10566,7 +10566,7 @@ pub unsafe fn vcvtq_low_bf16_f32(a: float32x4_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10601,7 +10601,7 @@ pub unsafe fn vcvtq_low_bf16_f32(a: float32x4_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10634,7 +10634,7 @@ pub unsafe fn vcvtq_low_bf16_f32(a: float32x4_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -10666,7 +10666,7 @@ pub unsafe fn vcvtq_low_bf16_f32(a: float32x4_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -11162,7 +11162,7 @@ pub fn vdotq_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x16_t) -> uint32x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -12297,7 +12297,7 @@ pub fn vdup_lane_u64(a: uint64x1_t) -> uint64x1_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -12324,7 +12324,7 @@ pub unsafe fn vdup_laneq_bf16(a: bfloat16x8_t) -> bfloat16x4_t #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -12353,7 +12353,7 @@ pub unsafe fn vdup_laneq_bf16(a: bfloat16x8_t) -> bfloat16x4_t #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -12380,7 +12380,7 @@ pub unsafe fn vdupq_laneq_bf16(a: bfloat16x8_t) -> bfloat16x8_t #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -13520,7 +13520,7 @@ pub fn vdup_laneq_u64(a: uint64x2_t) -> uint64x1_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -14081,7 +14081,7 @@ fn vdupq_n_f32_vfp4(value: f32) -> float32x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -14107,7 +14107,7 @@ pub unsafe fn vduph_lane_bf16(a: bfloat16x4_t) -> bfloat16_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -14133,7 +14133,7 @@ pub unsafe fn vduph_laneq_bf16(a: bfloat16x8_t) -> bfloat16_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -14367,7 +14367,7 @@ pub fn vdupq_laneq_u64(a: uint64x2_t) -> uint64x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -16572,7 +16572,7 @@ pub fn vfmsq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -16598,7 +16598,7 @@ pub unsafe fn vget_high_bf16(a: bfloat16x8_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -17434,7 +17434,7 @@ pub fn vget_high_u64(a: uint64x2_t) -> uint64x1_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -17461,7 +17461,7 @@ pub unsafe fn vget_lane_bf16(a: bfloat16x4_t) -> bfloat16_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -18533,7 +18533,7 @@ pub fn vget_lane_u64(v: uint64x1_t) -> u64 { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -19039,7 +19039,7 @@ pub fn vget_low_u64(a: uint64x2_t) -> uint64x1_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -19066,7 +19066,7 @@ pub unsafe fn vgetq_lane_bf16(a: bfloat16x8_t) -> bfloat16_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -19789,7 +19789,7 @@ pub fn vhsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -19814,7 +19814,7 @@ pub unsafe fn vld1_bf16(ptr: *const bfloat16_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -19839,7 +19839,7 @@ pub unsafe fn vld1q_bf16(ptr: *const bfloat16_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -19865,7 +19865,7 @@ pub unsafe fn vld1_bf16_x2(ptr: *const bfloat16_t) -> bfloat16x4x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -19891,7 +19891,7 @@ pub unsafe fn vld1_bf16_x3(ptr: *const bfloat16_t) -> bfloat16x4x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -19917,7 +19917,7 @@ pub unsafe fn vld1_bf16_x4(ptr: *const bfloat16_t) -> bfloat16x4x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -19943,7 +19943,7 @@ pub unsafe fn vld1q_bf16_x2(ptr: *const bfloat16_t) -> bfloat16x8x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -19969,7 +19969,7 @@ pub unsafe fn vld1q_bf16_x3(ptr: *const bfloat16_t) -> bfloat16x8x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -19995,7 +19995,7 @@ pub unsafe fn vld1q_bf16_x4(ptr: *const bfloat16_t) -> bfloat16x8x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] @@ -20024,7 +20024,7 @@ pub unsafe fn vld1_bf16_x2(ptr: *const bfloat16_t) -> bfloat16x4x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] @@ -20053,7 +20053,7 @@ pub unsafe fn vld1_bf16_x3(ptr: *const bfloat16_t) -> bfloat16x4x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] @@ -20082,7 +20082,7 @@ pub unsafe fn vld1_bf16_x4(ptr: *const bfloat16_t) -> bfloat16x4x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] @@ -20111,7 +20111,7 @@ pub unsafe fn vld1q_bf16_x2(ptr: *const bfloat16_t) -> bfloat16x8x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] @@ -20140,7 +20140,7 @@ pub unsafe fn vld1q_bf16_x3(ptr: *const bfloat16_t) -> bfloat16x8x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] @@ -20169,7 +20169,7 @@ pub unsafe fn vld1q_bf16_x4(ptr: *const bfloat16_t) -> bfloat16x8x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -20194,7 +20194,7 @@ pub unsafe fn vld1_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -21288,7 +21288,7 @@ pub unsafe fn vld1q_f32_x4(a: *const f32) -> float32x4x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -21316,7 +21316,7 @@ pub unsafe fn vld1_lane_bf16( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -23806,7 +23806,7 @@ pub unsafe fn vld1q_dup_p64(ptr: *const p64) -> poly64x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -23832,7 +23832,7 @@ pub unsafe fn vld2_bf16(ptr: *const bfloat16_t) -> bfloat16x4x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -23858,7 +23858,7 @@ pub unsafe fn vld2q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -23884,7 +23884,7 @@ pub unsafe fn vld3_bf16(ptr: *const bfloat16_t) -> bfloat16x4x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -23910,7 +23910,7 @@ pub unsafe fn vld3q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -23936,7 +23936,7 @@ pub unsafe fn vld4_bf16(ptr: *const bfloat16_t) -> bfloat16x4x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(target_arch = "arm")] #[cfg_attr( @@ -23962,7 +23962,7 @@ pub unsafe fn vld4q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld2.16"))] @@ -23991,7 +23991,7 @@ pub unsafe fn vld2_bf16(ptr: *const bfloat16_t) -> bfloat16x4x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld2.16"))] @@ -24020,7 +24020,7 @@ pub unsafe fn vld2q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld3.16"))] @@ -24049,7 +24049,7 @@ pub unsafe fn vld3_bf16(ptr: *const bfloat16_t) -> bfloat16x4x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld3.16"))] @@ -24078,7 +24078,7 @@ pub unsafe fn vld3q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld4.16"))] @@ -24107,7 +24107,7 @@ pub unsafe fn vld4_bf16(ptr: *const bfloat16_t) -> bfloat16x4x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg(not(target_arch = "arm"))] #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld4.16"))] @@ -24136,7 +24136,7 @@ pub unsafe fn vld4q_bf16(ptr: *const bfloat16_t) -> bfloat16x8x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -24161,7 +24161,7 @@ pub unsafe fn vld2_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x4x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -24186,7 +24186,7 @@ pub unsafe fn vld2q_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x8x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -24211,7 +24211,7 @@ pub unsafe fn vld3_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x4x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -24236,7 +24236,7 @@ pub unsafe fn vld3q_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x8x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -24261,7 +24261,7 @@ pub unsafe fn vld4_dup_bf16(ptr: *const bfloat16_t) -> bfloat16x4x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -25329,7 +25329,7 @@ pub unsafe fn vld2q_s32(a: *const i32) -> int32x4x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -25357,7 +25357,7 @@ pub unsafe fn vld2_lane_bf16( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -27290,7 +27290,7 @@ pub unsafe fn vld3q_s32(a: *const i32) -> int32x4x3_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -27318,7 +27318,7 @@ pub unsafe fn vld3_lane_bf16( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -29297,7 +29297,7 @@ pub unsafe fn vld4q_s32(a: *const i32) -> int32x4x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -29325,7 +29325,7 @@ pub unsafe fn vld4_lane_bf16( #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -44966,7 +44966,7 @@ pub fn vrecpsq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -44991,7 +44991,7 @@ pub unsafe fn vreinterpret_bf16_f16(a: float16x4_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45016,7 +45016,7 @@ pub unsafe fn vreinterpret_bf16_f32(a: float32x2_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45041,7 +45041,7 @@ pub unsafe fn vreinterpret_bf16_p16(a: poly16x4_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45066,7 +45066,7 @@ pub unsafe fn vreinterpret_bf16_p64(a: poly64x1_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45091,7 +45091,7 @@ pub unsafe fn vreinterpret_bf16_p8(a: poly8x8_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45116,7 +45116,7 @@ pub unsafe fn vreinterpret_bf16_s16(a: int16x4_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45141,7 +45141,7 @@ pub unsafe fn vreinterpret_bf16_s32(a: int32x2_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45166,7 +45166,7 @@ pub unsafe fn vreinterpret_bf16_s64(a: int64x1_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45191,7 +45191,7 @@ pub unsafe fn vreinterpret_bf16_s8(a: int8x8_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45216,7 +45216,7 @@ pub unsafe fn vreinterpret_bf16_u16(a: uint16x4_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45241,7 +45241,7 @@ pub unsafe fn vreinterpret_bf16_u32(a: uint32x2_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45266,7 +45266,7 @@ pub unsafe fn vreinterpret_bf16_u64(a: uint64x1_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45291,7 +45291,7 @@ pub unsafe fn vreinterpret_bf16_u8(a: uint8x8_t) -> bfloat16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45316,7 +45316,7 @@ pub unsafe fn vreinterpret_f16_bf16(a: bfloat16x4_t) -> float16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45341,7 +45341,7 @@ pub unsafe fn vreinterpret_f32_bf16(a: bfloat16x4_t) -> float32x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45366,7 +45366,7 @@ pub unsafe fn vreinterpret_p16_bf16(a: bfloat16x4_t) -> poly16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45391,7 +45391,7 @@ pub unsafe fn vreinterpret_p64_bf16(a: bfloat16x4_t) -> poly64x1_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45416,7 +45416,7 @@ pub unsafe fn vreinterpret_p8_bf16(a: bfloat16x4_t) -> poly8x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45441,7 +45441,7 @@ pub unsafe fn vreinterpret_s16_bf16(a: bfloat16x4_t) -> int16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45466,7 +45466,7 @@ pub unsafe fn vreinterpret_s32_bf16(a: bfloat16x4_t) -> int32x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45491,7 +45491,7 @@ pub unsafe fn vreinterpret_s64_bf16(a: bfloat16x4_t) -> int64x1_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45516,7 +45516,7 @@ pub unsafe fn vreinterpret_s8_bf16(a: bfloat16x4_t) -> int8x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45541,7 +45541,7 @@ pub unsafe fn vreinterpret_u16_bf16(a: bfloat16x4_t) -> uint16x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45566,7 +45566,7 @@ pub unsafe fn vreinterpret_u32_bf16(a: bfloat16x4_t) -> uint32x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -45591,7 +45591,7 @@ pub unsafe fn vreinterpret_u64_bf16(a: bfloat16x4_t) -> uint64x1_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47424,7 +47424,7 @@ pub fn vreinterpretq_u16_p16(a: poly16x8_t) -> uint16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47449,7 +47449,7 @@ pub unsafe fn vreinterpretq_bf16_f16(a: float16x8_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47474,7 +47474,7 @@ pub unsafe fn vreinterpretq_bf16_f32(a: float32x4_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47499,7 +47499,7 @@ pub unsafe fn vreinterpretq_bf16_p128(a: p128) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47524,7 +47524,7 @@ pub unsafe fn vreinterpretq_bf16_p16(a: poly16x8_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47549,7 +47549,7 @@ pub unsafe fn vreinterpretq_bf16_p64(a: poly64x2_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47574,7 +47574,7 @@ pub unsafe fn vreinterpretq_bf16_p8(a: poly8x16_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47599,7 +47599,7 @@ pub unsafe fn vreinterpretq_bf16_s16(a: int16x8_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47624,7 +47624,7 @@ pub unsafe fn vreinterpretq_bf16_s32(a: int32x4_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47649,7 +47649,7 @@ pub unsafe fn vreinterpretq_bf16_s64(a: int64x2_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47674,7 +47674,7 @@ pub unsafe fn vreinterpretq_bf16_s8(a: int8x16_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47699,7 +47699,7 @@ pub unsafe fn vreinterpretq_bf16_u16(a: uint16x8_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47724,7 +47724,7 @@ pub unsafe fn vreinterpretq_bf16_u32(a: uint32x4_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47749,7 +47749,7 @@ pub unsafe fn vreinterpretq_bf16_u64(a: uint64x2_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47774,7 +47774,7 @@ pub unsafe fn vreinterpretq_bf16_u8(a: uint8x16_t) -> bfloat16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47799,7 +47799,7 @@ pub unsafe fn vreinterpretq_f16_bf16(a: bfloat16x8_t) -> float16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47824,7 +47824,7 @@ pub unsafe fn vreinterpretq_f32_bf16(a: bfloat16x8_t) -> float32x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47849,7 +47849,7 @@ pub unsafe fn vreinterpretq_p128_bf16(a: bfloat16x8_t) -> p128 { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47874,7 +47874,7 @@ pub unsafe fn vreinterpretq_p16_bf16(a: bfloat16x8_t) -> poly16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47899,7 +47899,7 @@ pub unsafe fn vreinterpretq_p64_bf16(a: bfloat16x8_t) -> poly64x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47924,7 +47924,7 @@ pub unsafe fn vreinterpretq_p8_bf16(a: bfloat16x8_t) -> poly8x16_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47949,7 +47949,7 @@ pub unsafe fn vreinterpretq_s16_bf16(a: bfloat16x8_t) -> int16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47974,7 +47974,7 @@ pub unsafe fn vreinterpretq_s32_bf16(a: bfloat16x8_t) -> int32x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -47999,7 +47999,7 @@ pub unsafe fn vreinterpretq_s64_bf16(a: bfloat16x8_t) -> int64x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -48024,7 +48024,7 @@ pub unsafe fn vreinterpretq_s8_bf16(a: bfloat16x8_t) -> int8x16_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -48049,7 +48049,7 @@ pub unsafe fn vreinterpretq_u16_bf16(a: bfloat16x8_t) -> uint16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -48074,7 +48074,7 @@ pub unsafe fn vreinterpretq_u32_bf16(a: bfloat16x8_t) -> uint32x4_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -48099,7 +48099,7 @@ pub unsafe fn vreinterpretq_u64_bf16(a: bfloat16x8_t) -> uint64x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -57375,7 +57375,7 @@ pub fn vrsubhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -57402,7 +57402,7 @@ pub unsafe fn vset_lane_bf16(a: bfloat16_t, v: bfloat16x4_t) -> #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -57431,7 +57431,7 @@ pub unsafe fn vset_lane_bf16(a: bfloat16_t, v: bfloat16x4_t) -> #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -57458,7 +57458,7 @@ pub unsafe fn vsetq_lane_bf16(a: bfloat16_t, v: bfloat16x8_t) - #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -61898,7 +61898,7 @@ pub fn vsriq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -61923,7 +61923,7 @@ pub unsafe fn vst1_bf16(ptr: *mut bfloat16_t, val: bfloat16x4_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -61948,7 +61948,7 @@ pub unsafe fn vst1q_bf16(ptr: *mut bfloat16_t, val: bfloat16x8_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -61973,7 +61973,7 @@ pub unsafe fn vst1_bf16_x2(ptr: *mut bfloat16_t, val: bfloat16x4x2_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -61998,7 +61998,7 @@ pub unsafe fn vst1_bf16_x3(ptr: *mut bfloat16_t, val: bfloat16x4x3_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -62023,7 +62023,7 @@ pub unsafe fn vst1_bf16_x4(ptr: *mut bfloat16_t, val: bfloat16x4x4_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -62048,7 +62048,7 @@ pub unsafe fn vst1q_bf16_x2(ptr: *mut bfloat16_t, val: bfloat16x8x2_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -62073,7 +62073,7 @@ pub unsafe fn vst1q_bf16_x3(ptr: *mut bfloat16_t, val: bfloat16x8x3_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -62926,7 +62926,7 @@ pub unsafe fn vst1q_f32_x4(a: *mut f32, b: float32x4x4_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -62951,7 +62951,7 @@ pub unsafe fn vst1_lane_bf16(a: *mut bfloat16_t, v: bfloat16x4_ #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -65598,7 +65598,7 @@ pub unsafe fn vst1q_lane_p64(a: *mut p64, b: poly64x2_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -65623,7 +65623,7 @@ pub unsafe fn vst2_bf16(a: *mut bfloat16_t, b: bfloat16x4x2_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -65952,7 +65952,7 @@ pub unsafe fn vst2q_s32(a: *mut i32, b: int32x4x2_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -65977,7 +65977,7 @@ pub unsafe fn vst2_lane_bf16(a: *mut bfloat16_t, v: bfloat16x4x #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -66863,7 +66863,7 @@ pub unsafe fn vst2q_p16(a: *mut p16, b: poly16x8x2_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -66888,7 +66888,7 @@ pub unsafe fn vst3_bf16(a: *mut bfloat16_t, b: bfloat16x4x3_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -67185,7 +67185,7 @@ pub unsafe fn vst3q_s32(a: *mut i32, b: int32x4x3_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -67210,7 +67210,7 @@ pub unsafe fn vst3_lane_bf16(a: *mut bfloat16_t, v: bfloat16x4x #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -68152,7 +68152,7 @@ pub unsafe fn vst3q_p16(a: *mut p16, b: poly16x8x3_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -68177,7 +68177,7 @@ pub unsafe fn vst4_bf16(a: *mut bfloat16_t, b: bfloat16x4x4_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -68569,7 +68569,7 @@ pub unsafe fn vst4q_s32(a: *mut i32, b: int32x4x4_t) { #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", @@ -68594,7 +68594,7 @@ pub unsafe fn vst4_lane_bf16(a: *mut bfloat16_t, v: bfloat16x4x #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr( not(target_arch = "arm"), - unstable(feature = "stdarch_neon_bf16", issue = "none") + unstable(feature = "stdarch_neon_bf16", issue = "164986") )] #[cfg_attr( target_arch = "arm", diff --git a/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml b/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml index a1c955bc5c..4cb43ee580 100644 --- a/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml +++ b/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml @@ -63,9 +63,9 @@ aarch64-crc-stable: &aarch64-crc-stable neon-unstable-f16: &neon-unstable-f16 FnCall: [unstable, ['feature = "stdarch_neon_f16"', 'issue = "136306"']] -# #[unstable(feature = "stdarch_neon_bf16", issue = "none")] +# #[unstable(feature = "stdarch_neon_bf16", issue = "164986")] neon-unstable-bf16: &neon-unstable-bf16 - FnCall: [unstable, ['feature = "stdarch_neon_bf16"', 'issue = "none"']] + FnCall: [unstable, ['feature = "stdarch_neon_bf16"', 'issue = "164986"']] # #[cfg(any(target_has_reliable_f16b, doc))] cfg-reliable-bf16: &cfg-reliable-bf16 diff --git a/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml b/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml index e80c4b2f8d..de352e3930 100644 --- a/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml +++ b/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml @@ -136,9 +136,9 @@ neon-unstable-f16: &neon-unstable-f16 all-neon-target-aarch64-arm64ec-little-endian: &all-neon-target-aarch64-arm64ec-little-endian FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}, 'target_endian = "little"']] -# #[cfg_attr(not(target_arch = "arm"), unstable(feature = "stdarch_neon_bf16", issue = "none"))] +# #[cfg_attr(not(target_arch = "arm"), unstable(feature = "stdarch_neon_bf16", issue = "164986"))] neon-unstable-bf16: &neon-unstable-bf16 - FnCall: [cfg_attr, [*not-arm, {FnCall: [unstable, ['feature = "stdarch_neon_bf16"', 'issue = "none"']]}]] + FnCall: [cfg_attr, [*not-arm, {FnCall: [unstable, ['feature = "stdarch_neon_bf16"', 'issue = "164986"']]}]] # #[cfg(any(target_has_reliable_f16b, doc))] cfg-reliable-bf16: &cfg-reliable-bf16