1011 lines
39 KiB
Rust
1011 lines
39 KiB
Rust
use core::intrinsics::const_eval_select;
|
|
use core::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount};
|
|
|
|
use crate::util;
|
|
|
|
use util::cast;
|
|
|
|
const W_128: usize = 128 / 8;
|
|
const W_256: usize = 256 / 8;
|
|
const W_512: usize = 512 / 8;
|
|
|
|
/// The value which cause `vpshufb` to write 0 instead of indexing.
|
|
const SHUF_0: u8 = 0b1000_0000;
|
|
|
|
#[cfg(target_arch = "aarch64")]
|
|
pub use core::arch::aarch64 as arch;
|
|
#[cfg(target_arch = "arm")]
|
|
pub use core::arch::arm as arch;
|
|
#[cfg(target_arch = "wasm32")]
|
|
pub use core::arch::wasm32 as arch;
|
|
#[cfg(target_arch = "wasm64")]
|
|
pub use core::arch::wasm64 as arch;
|
|
#[cfg(target_arch = "x86")]
|
|
pub use core::arch::x86 as arch;
|
|
#[cfg(target_arch = "x86_64")]
|
|
pub use core::arch::x86_64 as arch;
|
|
|
|
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
|
|
pub use arch::{__m128i, __m256i, __m512i};
|
|
|
|
#[cfg(feature = "trace-simd")]
|
|
#[macro_export]
|
|
macro_rules! __if_trace_simd {
|
|
($( $tt:tt )*) => { { $( $tt )* } };
|
|
}
|
|
#[cfg(not(feature = "trace-simd"))]
|
|
#[macro_export]
|
|
macro_rules! __if_trace_simd {
|
|
($( $tt:tt )*) => {};
|
|
}
|
|
|
|
pub use __if_trace_simd as if_trace_simd;
|
|
|
|
pub trait IsSimd {
|
|
type Lane;
|
|
|
|
const LANES: usize;
|
|
}
|
|
|
|
impl<T, const LANES: usize> IsSimd for Simd<T, LANES>
|
|
where
|
|
LaneCount<LANES>: SupportedLaneCount,
|
|
T: SimdElement,
|
|
{
|
|
type Lane = T;
|
|
|
|
const LANES: usize = LANES;
|
|
}
|
|
|
|
/*macro_rules! specialized {
|
|
($(
|
|
$vis:vis fn $name:ident<$LANES:ident$(, [$( $generics:tt )+])?>($( $args:tt )*) -> $rt:ty $(where [ $( $where:tt )* ])? {
|
|
$(
|
|
$width:pat_param $( if $cfg:meta )? => $impl:expr
|
|
),+
|
|
$(,)?
|
|
}
|
|
)+) => {$(
|
|
#[allow(dead_code)]
|
|
#[inline(always)]
|
|
$vis fn $name<const $LANES: usize$(, $( $generics )+)?>($( $args )*) -> $rt $( where $( $where )* )? {
|
|
// abusing const generics to specialize without the unsoundness of real specialization!
|
|
match $LANES {
|
|
$(
|
|
$( #[cfg( $cfg )] )?
|
|
$width => $impl
|
|
),+
|
|
}
|
|
}
|
|
)+};
|
|
($LANES:ident =>
|
|
$(
|
|
fn $name:ident$(<[$( $generics:tt )+]>)?($( $args:tt )*) -> $rt:ty $(where [ $( $where:tt )* ])? {
|
|
$(
|
|
$width:pat_param $( if $cfg:meta )? => $impl:expr
|
|
),+
|
|
$(,)?
|
|
}
|
|
)+
|
|
) => {
|
|
$(
|
|
#[inline(always)]
|
|
fn $name$(<$( $generics )+>)?($( $args )*) -> $rt $( where $( $where )* )? {
|
|
// abusing const generics to specialize without the unsoundness of real specialization!
|
|
match $LANES {
|
|
$(
|
|
$( #[cfg( $cfg )] )?
|
|
$width => $impl
|
|
),+
|
|
}
|
|
}
|
|
)+
|
|
};
|
|
($trait:ident for $ty:ty;
|
|
$(
|
|
fn $name:ident<$LANES:ident$(, [$( $generics:tt )+])?>($( $args:tt )*) -> $rt:ty $(where [ $( $where:tt )* ])? {
|
|
$(
|
|
$width:pat_param $( if $cfg:meta )? => $impl:expr
|
|
),+
|
|
$(,)?
|
|
}
|
|
)+
|
|
) => {
|
|
impl<const $LANES: usize> $trait for $ty {$(
|
|
specialized! { LANES =>
|
|
fn $name$(<[$( $generics:tt )+]>)?($( $args )*) -> $rt $(where [$( $where )*])? {
|
|
$(
|
|
$width $( if $cfg )? => $impl
|
|
),+
|
|
}
|
|
}
|
|
)+}
|
|
};
|
|
}*/
|
|
|
|
macro_rules! specialized {
|
|
($ty:ty =>
|
|
$(
|
|
$vis:vis fn $name:ident<$LANES:ident$(, [$( $generics:tt )+])?>($( $args:tt )*) -> $rt:ty $(where [ $( $where:tt )* ])? {
|
|
$(
|
|
$width:pat_param $( if $cfg:meta )? => $impl:expr
|
|
),+
|
|
$(,)?
|
|
}
|
|
)+
|
|
) => {$(
|
|
#[allow(dead_code)]
|
|
#[inline(always)]
|
|
$vis fn $name<const $LANES: usize$(, $( $generics )+)?>($( $args )*) -> $rt $( where $( $where )* )? {
|
|
// abusing const generics to specialize without the unsoundness of real specialization!
|
|
match $LANES * core::mem::size_of::<$ty>() {
|
|
$(
|
|
$( #[cfg( $cfg )] )?
|
|
$width => $impl
|
|
),+
|
|
}
|
|
}
|
|
)+};
|
|
}
|
|
|
|
macro_rules! set1_short {
|
|
($inst:ident, $vec:ident, $reg:ident, $n:ident: $n_ty:ty) => {{
|
|
// WOW. this is 12% faster than broadcast on xmm. Seems not so much on ymm (more expensive
|
|
// array init?).
|
|
const O_LANES: usize = core::mem::size_of::<$vec>() / core::mem::size_of::<$n_ty>();
|
|
util::cast::<_, $vec>(Simd::<$n_ty, O_LANES>::from_array([$n; O_LANES]))
|
|
//let out: $vec;
|
|
//core::arch::asm!(concat!(stringify!($inst), " {}, {}"), lateout($reg) out, in(xmm_reg) cast::<_, __m128i>($n), options(pure, nomem, preserves_flags, nostack));
|
|
//out
|
|
}};
|
|
}
|
|
|
|
macro_rules! set1_long {
|
|
($inst:ident, $vec:ident, $reg:ident, $n:ident: $n_ty:ty) => {{
|
|
fn runtime(n: $n_ty) -> $vec {
|
|
unsafe {
|
|
//const O_LANES: usize = core::mem::size_of::<$vec>() / core::mem::size_of::<$n_ty>();
|
|
//util::cast::<_, $vec>(Simd::<$n_ty, O_LANES>::from_array([$n; O_LANES]))
|
|
let out: $vec;
|
|
core::arch::asm!(concat!(stringify!($inst), " {}, {}"), lateout($reg) out, in(xmm_reg) cast::<_, __m128i>(n), options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}
|
|
const fn compiletime(n: $n_ty) -> $vec {
|
|
const O_LANES: usize = core::mem::size_of::<$vec>() / core::mem::size_of::<$n_ty>();
|
|
unsafe { util::cast::<_, $vec>(Simd::<$n_ty, O_LANES>::from_array([n; O_LANES])) }
|
|
}
|
|
const_eval_select(($n,), compiletime, runtime)
|
|
}};
|
|
}
|
|
|
|
pub trait DoubleWidth {
|
|
type Output;
|
|
}
|
|
|
|
macro_rules! impl_double_width {
|
|
($($in:ty => $out:ty),+) => {
|
|
$(
|
|
impl DoubleWidth for $in {
|
|
type Output = $out;
|
|
}
|
|
)+
|
|
}
|
|
}
|
|
|
|
impl_double_width!(u8 => u16, u16 => u32, u32 => u64);
|
|
|
|
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
|
|
impl_double_width!(u64 => arch::__m128i, arch::__m128i => arch::__m256i, arch::__m256i => arch::__m512i);
|
|
|
|
#[const_trait]
|
|
pub trait SimdSplat {
|
|
type Output<const LANES: usize> where LaneCount<LANES>: SupportedLaneCount;
|
|
|
|
fn splat<const LANES: usize>(self) -> Self::Output<LANES> where LaneCount<LANES>: SupportedLaneCount;
|
|
|
|
fn splat_zero<const LANES: usize>() -> Self::Output<LANES> where LaneCount<LANES>: SupportedLaneCount;
|
|
}
|
|
|
|
pub trait SimdLoad: Sized + SimdElement {
|
|
fn load_128(self) -> arch::__m128i;
|
|
|
|
#[inline(always)]
|
|
fn load_256(self) -> arch::__m256i {
|
|
unsafe { util::cast(self.load_128()) }
|
|
}
|
|
|
|
#[inline(always)]
|
|
fn load_512(self) -> arch::__m512i {
|
|
unsafe { util::cast(self.load_128()) }
|
|
}
|
|
}
|
|
|
|
pub trait SimdLoadArray: Sized + SimdElement {
|
|
fn load_array<const LANES: usize>(array: [Self; LANES]) -> Simd<Self, LANES> where LaneCount<LANES>: SupportedLaneCount;
|
|
}
|
|
|
|
impl SimdLoadArray for u64 {
|
|
specialized! { Self =>
|
|
fn load_array<LANES>(array: [Self; LANES]) -> Simd<Self, LANES> where [LaneCount<LANES>: SupportedLaneCount] {
|
|
W_128 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx") => unsafe {
|
|
let mut out: __m128i;
|
|
core::arch::asm!("vmovq {}, {:r}", lateout(xmm_reg) out, in(reg) array[0]);
|
|
core::arch::asm!("vpinsrq {out}, {out}, {b:r}, 0x1", out = inout(xmm_reg) out, b = in(reg) array[1]);
|
|
cast(out)
|
|
},
|
|
W_256 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx") => unsafe {
|
|
let mut out_a: __m128i;
|
|
core::arch::asm!("vmovq {}, {:r}", lateout(xmm_reg) out_a, in(reg) array[0]);
|
|
core::arch::asm!("vpinsrq {out}, {out}, {b:r}, 0x1", out = inout(xmm_reg) out_a, b = in(reg) array[1]);
|
|
let mut out_b: __m128i;
|
|
core::arch::asm!("vmovq {}, {:r}", lateout(xmm_reg) out_b, in(reg) array[2]);
|
|
core::arch::asm!("vpinsrq {out}, {out}, {b:r}, 0x1", out = inout(xmm_reg) out_b, b = in(reg) array[3]);
|
|
cast(merge_m128_m256(out_a, out_b))
|
|
},
|
|
_ => Simd::from_array(array),
|
|
}
|
|
}
|
|
}
|
|
|
|
macro_rules! impl_load {
|
|
(($in:ident, $($in_fmt:ident)?) $in_v:ident) => {{
|
|
unsafe {
|
|
let out: _;
|
|
core::arch::asm!(concat!("vmovq {}, {:", $(stringify!($in_fmt), )? "}"), lateout(xmm_reg) out, in($in) $in_v, options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}}
|
|
}
|
|
|
|
macro_rules! simd_load_fallback {
|
|
($ty:ty, $vec_ty:ty, $self:ident) => {{
|
|
let mut a = core::mem::MaybeUninit::uninit_array();
|
|
a[0] = core::mem::MaybeUninit::new($self);
|
|
for i in 1..(core::mem::size_of::<$vec_ty>() / core::mem::size_of::<$ty>()) {
|
|
a[i] = core::mem::MaybeUninit::new(0);
|
|
}
|
|
let a = unsafe { core::mem::MaybeUninit::array_assume_init(a) };
|
|
Simd::from_array(a).into()
|
|
}};
|
|
}
|
|
|
|
macro_rules! impl_ops {
|
|
($(
|
|
$ty:ty {
|
|
/// The appropriate register type.
|
|
reg: $reg:ident,
|
|
reg_fmt: $($reg_fmt:ident)?,
|
|
broadcast: $broadcast:ident,
|
|
set1: [$set1_128:ident, $set1_256:ident, $set1_512:ident]
|
|
$(,)?
|
|
}
|
|
)+) => {$(
|
|
impl SimdLoad for $ty {
|
|
#[inline(always)]
|
|
fn load_128(self) -> arch::__m128i {
|
|
#[cfg(target_feature = "avx")]
|
|
{ impl_load!(($reg, $($reg_fmt)?) self) }
|
|
#[cfg(not(target_feature = "avx"))]
|
|
simd_load_fallback!($ty, arch::__m128i, self)
|
|
}
|
|
}
|
|
|
|
impl const SimdSplat for $ty {
|
|
type Output<const LANES: usize> = Simd<$ty, LANES> where LaneCount<LANES>: SupportedLaneCount;
|
|
|
|
specialized! { Self =>
|
|
fn splat<LANES>(self) -> Self::Output<LANES> where [LaneCount<LANES>: SupportedLaneCount] {
|
|
W_128 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx2") => unsafe { cast(set1_short!($broadcast, __m128i, xmm_reg, self: $ty)) },
|
|
W_256 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx2") => unsafe { cast(set1_long!($broadcast, __m256i, ymm_reg, self: $ty)) },
|
|
W_512 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx512bw") => unsafe { cast(set1_long!($broadcast, __m512i, zmm_reg, self: $ty)) },
|
|
|
|
// these are *terrible*. They compile to a bunch of MOVs and SETs
|
|
W_128 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "sse2", not(target_feature = "avx2")) => unsafe { cast(arch::$set1_128(self as i8)) },
|
|
W_256 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx", not(target_feature = "avx2")) => unsafe { cast(arch::$set1_256(self as i8)) },
|
|
|
|
// I can't actually test these, but they're documented as doing either a broadcast or the terrible approach mentioned above.
|
|
W_512 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx512f") => unsafe { cast(arch::$set1_512(self as i8)) },
|
|
_ => {
|
|
#[inline(always)]
|
|
const fn compiletime<const LANES: usize>(_: $ty) -> Simd<$ty, LANES>
|
|
where
|
|
LaneCount<LANES>: SupportedLaneCount,
|
|
{
|
|
panic!("unsupported compile-time splat");
|
|
}
|
|
#[inline(always)]
|
|
fn runtime<const LANES: usize>(v: $ty) -> Simd<$ty, LANES>
|
|
where
|
|
LaneCount<LANES>: SupportedLaneCount,
|
|
{
|
|
Simd::splat(v)
|
|
}
|
|
unsafe { const_eval_select((self,), compiletime, runtime) }
|
|
},
|
|
}
|
|
|
|
fn splat_zero<LANES>() -> Self::Output<LANES> where [LaneCount<LANES>: SupportedLaneCount] {
|
|
W_128 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "sse2") => {
|
|
#[inline(always)]
|
|
const fn compiletime<const LANES: usize>() -> Simd<$ty, LANES>
|
|
where
|
|
LaneCount<LANES>: SupportedLaneCount,
|
|
{
|
|
<$ty>::splat(0)
|
|
}
|
|
#[inline(always)]
|
|
fn runtime<const LANES: usize>() -> Simd<$ty, LANES>
|
|
where
|
|
LaneCount<LANES>: SupportedLaneCount,
|
|
{
|
|
unsafe { cast(arch::_mm_setzero_si128()) }
|
|
}
|
|
unsafe { const_eval_select((), compiletime, runtime) }
|
|
},
|
|
W_256 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx") => {
|
|
#[inline(always)]
|
|
const fn compiletime<const LANES: usize>() -> Simd<$ty, LANES>
|
|
where
|
|
LaneCount<LANES>: SupportedLaneCount,
|
|
{
|
|
<$ty>::splat(0)
|
|
}
|
|
#[inline(always)]
|
|
fn runtime<const LANES: usize>() -> Simd<$ty, LANES>
|
|
where
|
|
LaneCount<LANES>: SupportedLaneCount,
|
|
{
|
|
unsafe { cast(arch::_mm256_setzero_si256()) }
|
|
}
|
|
unsafe { const_eval_select((), compiletime, runtime) }
|
|
},
|
|
W_512 if all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx512f") => {
|
|
#[inline(always)]
|
|
const fn compiletime<const LANES: usize>() -> Simd<$ty, LANES>
|
|
where
|
|
LaneCount<LANES>: SupportedLaneCount,
|
|
{
|
|
<$ty>::splat(0)
|
|
}
|
|
#[inline(always)]
|
|
fn runtime<const LANES: usize>() -> Simd<$ty, LANES>
|
|
where
|
|
LaneCount<LANES>: SupportedLaneCount,
|
|
{
|
|
unsafe { cast(arch::_mm512_setzero_si512()) }
|
|
}
|
|
unsafe { const_eval_select((), compiletime, runtime) }
|
|
},
|
|
_ => Self::splat(0),
|
|
}
|
|
}
|
|
}
|
|
)+};
|
|
}
|
|
|
|
impl_ops! {
|
|
u8 {
|
|
reg: reg_byte,
|
|
reg_fmt: ,
|
|
broadcast: vpbroadcastb,
|
|
set1: [_mm_set1_epi8, _mm256_set1_epi8, _mm512_set1_epi8],
|
|
}
|
|
|
|
u16 {
|
|
reg: reg,
|
|
reg_fmt: e,
|
|
broadcast: vpbroadcastw,
|
|
set1: [_mm_set1_epi16, _mm256_set1_epi16, _mm512_set1_epi16],
|
|
}
|
|
|
|
u32 {
|
|
reg: reg,
|
|
reg_fmt: e,
|
|
broadcast: vpbroadcastd,
|
|
set1: [_mm_set1_epi32, _mm256_set1_epi32, _mm512_set1_epi32],
|
|
}
|
|
|
|
u64 {
|
|
reg: reg,
|
|
reg_fmt: r,
|
|
broadcast: vpbroadcastq,
|
|
set1: [_mm_set1_epi64, _mm256_set1_epi64, _mm512_set1_epi64],
|
|
}
|
|
}
|
|
|
|
/// Defines the indices used by [`swizzle`].
|
|
#[macro_export]
|
|
macro_rules! __swizzle_indices {
|
|
($name:ident = [$( $index:literal ),+] $( , [$( $padding:tt )+] )?) => {
|
|
core::arch::global_asm!(concat!(".", stringify!($name), ":")
|
|
$( , concat!("\n .byte ", stringify!($index)) )+
|
|
$( $( , $crate::util::subst!([$padding], ["\n .zero 1"]) )+ )?);
|
|
};
|
|
}
|
|
|
|
#[macro_export]
|
|
macro_rules! __swizzle {
|
|
/*(xmm_reg, $src:expr, $dest:expr, [$( $index:literal ),+] $( , [$( $padding:tt )+] )?) => {
|
|
$crate::simd::swizzle!(@ xmm_reg, $src, $dest, (xmmword) [$( $index ),+] $( , [$( $padding )+] )?)
|
|
};
|
|
(ymm_reg, $src:expr, $dest:expr, [$( $index:literal ),+] $( , [$( $padding:tt )+] )?) => {
|
|
$crate::simd::swizzle!(@ ymm_reg, $src, $dest, (ymmword) [$( $index ),+] $( , [$( $padding )+] )?)
|
|
};
|
|
(zmm_reg, $src:expr, $dest:expr, [$( $index:literal ),+] $( , [$( $padding:tt )+] )?) => {
|
|
$crate::simd::swizzle!(@ zmm_reg, $src, $dest, (zmmword) [$( $index ),+] $( , [$( $padding )+] )?)
|
|
};*/
|
|
(xmm_reg, $src:expr, $mode:ident $dest:expr, $indices:ident) => {
|
|
$crate::simd::swizzle!(@ xmm_reg, x, $src, $mode $dest, (xmmword) $indices)
|
|
};
|
|
(ymm_reg, $src:expr, $mode:ident $dest:expr, $indices:ident) => {
|
|
$crate::simd::swizzle!(@ ymm_reg, y, $src, $mode $dest, (ymmword) $indices)
|
|
};
|
|
(zmm_reg, $src:expr, $mode:ident $dest:expr, $indices:ident) => {
|
|
$crate::simd::swizzle!(@ zmm_reg z, $src, $mode $dest, (zmmword) $indices)
|
|
};
|
|
($reg:ident, $src:expr, $dest:expr, mem $indices:expr) => {
|
|
core::arch::asm!("vpshufb {}, {}, [{}]", in($reg) $src, lateout($reg) $dest, in(reg) $indices, options(readonly, preserves_flags, nostack));
|
|
};
|
|
($reg:ident, $src:expr, $dest:expr, data($indices_reg:ident) $indices:expr) => {
|
|
core::arch::asm!("vpshufb {}, {}, {}", in($reg) $src, lateout($reg) $dest, in($indices_reg) $indices, options(pure, nomem, preserves_flags, nostack));
|
|
};
|
|
//(@ $reg:ident, $src:expr, $dest:expr, ($indices_reg:ident) [$( $index:literal ),+] $( , [$( $padding:tt )+] )?) => {
|
|
(@ $reg:ident, $token:ident, $src:expr, $mode:ident $dest:expr, ($indices_reg:ident) $indices:ident) => {
|
|
core::arch::asm!(concat!("vpshufb {:", stringify!($token), "}, {:", stringify!($token), "}, ", stringify!($indices_reg), " ptr [rip + .", stringify!($indices), "]"), $mode($reg) $dest, in($reg) $src, options(pure, nomem, preserves_flags, nostack));
|
|
// core::arch::asm!("2:"
|
|
// $( , concat!("\n .byte ", stringify!($index)) )+
|
|
// $( $( , $crate::util::subst!([$padding], ["\n .zero 1"]) )+ )?
|
|
// , "\n3:\n", concat!(" vpshufb {}, {}, ", stringify!($indices_reg), " ptr [rip + 2b]"), in($reg) $src, lateout($reg) $dest)
|
|
};
|
|
// ($src:expr, $dest:expr, [$( $index:literal ),+] $( , [$( $padding:tt )+] )?) => {
|
|
// $crate::simd::swizzle!(@ $src, $dest, [$( stringify!($index) ),+] $( , [$( "\n ", subst!($padding, ""), "zero 1" )+] )?)
|
|
// };
|
|
// (@ $src:expr, $dest:expr, [$( $index:literal ),+] $( , [$( $padding:literal )+] )?) => {
|
|
// core::arch::asm!(r#"
|
|
// .indices:"#,
|
|
// $( "\n .byte ", $index ),+
|
|
// $( $( $padding ),+ )?
|
|
// r#"
|
|
// lsb:
|
|
// vpshufb {}, {}, xmmword ptr [rip + .indices]
|
|
// "#, in(xmm_reg) $src, lateout(xmm_reg) $dest)
|
|
// };
|
|
}
|
|
|
|
pub use __swizzle as swizzle;
|
|
pub use __swizzle_indices as swizzle_indices;
|
|
|
|
#[inline(always)]
|
|
pub fn interleave_lo_bytes_m128(a: arch::__m128i, b: arch::__m128i) -> arch::__m128i {
|
|
unsafe {
|
|
let out: _;
|
|
core::arch::asm!("vpunpcklbw {}, {}, {}", lateout(xmm_reg) out, in(xmm_reg) a, in(xmm_reg) b, options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}
|
|
|
|
#[inline(always)]
|
|
pub fn interleave_lo_quads_m128(a: arch::__m128i, b: arch::__m128i) -> arch::__m128i {
|
|
unsafe {
|
|
let out: _;
|
|
core::arch::asm!("vpunpcklqdq {}, {}, {}", lateout(xmm_reg) out, in(xmm_reg) a, in(xmm_reg) b, options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}
|
|
|
|
#[inline(always)]
|
|
pub fn interleave_lo_bytes_m256(a: arch::__m256i, b: arch::__m256i) -> arch::__m256i {
|
|
unsafe {
|
|
let out: _;
|
|
core::arch::asm!("vpunpcklbw {}, {}, {}", lateout(ymm_reg) out, in(ymm_reg) a, in(ymm_reg) b, options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}
|
|
|
|
#[inline(always)]
|
|
pub fn interleave_lo_quads_m256(a: arch::__m256i, b: arch::__m256i) -> arch::__m256i {
|
|
unsafe {
|
|
let out: _;
|
|
core::arch::asm!("vpunpcklqdq {}, {}, {}", lateout(ymm_reg) out, in(ymm_reg) a, in(ymm_reg) b, options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}
|
|
|
|
/// Merges the low halves of `a` and `b` into a single register like `ab`.
|
|
#[inline(always)]
|
|
pub fn merge_lo_hi_m128(a: arch::__m128i, b: arch::__m128i) -> arch::__m128i {
|
|
// this works because vpunpcklqdq interleaves the low order **quadwords** (aka the entire lower
|
|
// half)
|
|
// xmm0 = xmm1[0],xmm0[0]
|
|
interleave_lo_quads_m128(a, b)
|
|
}
|
|
|
|
/// The args are in little endian order (first arg is lowest order)
|
|
#[inline(always)]
|
|
pub fn merge_m128_m256(a: arch::__m128i, b: arch::__m128i) -> arch::__m256i {
|
|
unsafe {
|
|
let out: _;
|
|
core::arch::asm!("vinserti128 {}, {:y}, {}, 0x1", lateout(ymm_reg) out, in(ymm_reg) a, in(xmm_reg) b, options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}
|
|
|
|
#[inline(always)]
|
|
pub fn extract_hi_half(v: arch::__m256i) -> arch::__m128i {
|
|
unsafe {
|
|
arch::_mm256_extracti128_si256(v, 1)
|
|
}
|
|
}
|
|
|
|
macro_rules! extract_lohi_bytes {
|
|
(($mask:expr, $op12:ident, $op3:ident), $in:ident) => {{
|
|
const MASK: arch::__m128i = unsafe { core::mem::transmute($mask) };
|
|
unsafe {
|
|
let out: _;
|
|
core::arch::asm!(
|
|
//concat!("vmovdqa {mask}, xmmword ptr [rip + .", stringify!($mask), "]"),
|
|
"vextracti128 {inter}, {input:y}, 1",
|
|
concat!(stringify!($op12), " {inter}, {inter}, {mask}"),
|
|
concat!(stringify!($op12), " {output:x}, {input:x}, {mask}"),
|
|
concat!(stringify!($op3), " {output:x}, {output:x}, {inter}"),
|
|
mask = in(xmm_reg) MASK, input = in(ymm_reg) $in, output = lateout(xmm_reg) out, inter = out(xmm_reg) _,
|
|
options(pure, nomem, preserves_flags, nostack)
|
|
);
|
|
out
|
|
}
|
|
}};
|
|
}
|
|
|
|
#[inline(always)]
|
|
pub fn extract_lo_bytes(v: __m256i) -> __m128i {
|
|
extract_lohi_bytes!(([0x00ffu16; 8], vpand, vpackuswb), v)
|
|
}
|
|
|
|
#[inline(always)]
|
|
pub fn extract_hi_bytes(v: __m256i) -> __m128i {
|
|
extract_lohi_bytes!(
|
|
(
|
|
[0x1u8, 0x3, 0x5, 0x7, 0x9, 0xb, 0xd, 0xf, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0],
|
|
vpshufb,
|
|
vpunpcklqdq
|
|
),
|
|
v
|
|
)
|
|
}
|
|
|
|
pub trait SimdTestAnd {
|
|
/// Returns true if the result of the bitwise AND of `self` and `mask` is not all zero.
|
|
fn test_and_non_zero(self, mask: Self) -> bool;
|
|
}
|
|
|
|
pub trait SimdBitwise {
|
|
/// Returns the bitwise OR of `self` and `rhs`.
|
|
fn or(self, rhs: Self) -> Self;
|
|
|
|
/// Returns the bitwise AND of `self` and `rhs`.
|
|
fn and(self, rhs: Self) -> Self;
|
|
}
|
|
|
|
pub trait SimdWiden {
|
|
/// Widens the lower bytes by spacing and zero-extending them to N bytes.
|
|
fn widen<const N: usize, const O: usize, const BE: bool>(self) -> Self;
|
|
}
|
|
|
|
#[inline(always)]
|
|
pub fn interleave_m64(a: __m128i, b: __m128i) -> __m128i {
|
|
interleave_lo_bytes_m128(a, b)
|
|
}
|
|
|
|
#[inline(always)]
|
|
pub fn interleave_m128(a: __m128i, b: __m128i) -> __m256i {
|
|
const INTERLEAVE_A: Simd<u8, 32> = Simd::from_array(util::array_op!(gen[32] |i| {
|
|
if i & 1 == 0 {
|
|
(i as u8) >> 1
|
|
} else {
|
|
0xff
|
|
}
|
|
}));
|
|
const INTERLEAVE_B: Simd<u8, 32> = Simd::from_array(util::array_op!(gen[32] |i| {
|
|
if i & 1 == 0 {
|
|
0xff
|
|
} else {
|
|
(i as u8) >> 1
|
|
}
|
|
}));
|
|
unsafe {
|
|
//let zero: __m128i = u8::splat_zero::<16>().into();
|
|
let spaced_a = arch::_mm256_cvtepu8_epi16(a);
|
|
//let spaced_a = interleave_lo_bytes_m256(a, zero);
|
|
//let spaced_b = interleave_lo_bytes_m256(zero, b);
|
|
//let spaced_b = arch::_mm256_cvtepu8_epi16(b);
|
|
//let a = merge_m128_m256(a, a);
|
|
let b = merge_m128_m256(b, b);
|
|
//let spaced_a = arch::_mm256_shuffle_epi8(a, INTERLEAVE_A.into());
|
|
let spaced_b = arch::_mm256_shuffle_epi8(b, INTERLEAVE_B.into());
|
|
//let spaced_b: arch::__m256i = shl!(64, 8, (ymm_reg) spaced_b);
|
|
//interleave_lo_bytes_m256(a, b)
|
|
//interleave_lo_bytes_m256(spaced_a, spaced_b)
|
|
spaced_a.or(spaced_b)
|
|
}
|
|
}
|
|
|
|
#[cfg(target_feature = "avx")]
|
|
impl SimdTestAnd for arch::__m128i {
|
|
#[inline(always)]
|
|
fn test_and_non_zero(self, mask: Self) -> bool {
|
|
unsafe {
|
|
let out: u8;
|
|
core::arch::asm!("vptest {a}, {b}", "jz 2f", "mov {out}, 1", "jnz 3f", "2:", "mov {out}, 0", "3:", a = in(xmm_reg) self, b = in(xmm_reg) mask, out = lateout(reg_byte) out, options(pure, nomem, nostack));
|
|
core::mem::transmute(out)
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(target_feature = "avx")]
|
|
impl SimdTestAnd for arch::__m256i {
|
|
#[inline(always)]
|
|
fn test_and_non_zero(self, mask: Self) -> bool {
|
|
unsafe {
|
|
let out: u8;
|
|
core::arch::asm!("vptest {a}, {b}", "jz 2f", "mov {out}, 1", "jnz 3f", "2:", "mov {out}, 0", "3:", a = in(ymm_reg) self, b = in(ymm_reg) mask, out = lateout(reg_byte) out, options(pure, nomem, nostack));
|
|
core::mem::transmute(out)
|
|
}
|
|
}
|
|
}
|
|
|
|
const USE_BITWISE_INTRINSICS: bool = true;
|
|
|
|
#[cfg(target_feature = "avx")]
|
|
impl SimdBitwise for arch::__m128i {
|
|
#[inline(always)]
|
|
fn or(self, rhs: Self) -> Self {
|
|
unsafe {
|
|
if USE_BITWISE_INTRINSICS {
|
|
arch::_mm_or_si128(self, rhs)
|
|
} else {
|
|
let out: _;
|
|
core::arch::asm!("vpor {out}, {a}, {b}", a = in(xmm_reg) self, b = in(xmm_reg) rhs, out = lateout(xmm_reg) out, options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}
|
|
}
|
|
|
|
#[inline(always)]
|
|
fn and(self, rhs: Self) -> Self {
|
|
unsafe {
|
|
if USE_BITWISE_INTRINSICS {
|
|
arch::_mm_and_si128(self, rhs)
|
|
} else {
|
|
let out: _;
|
|
core::arch::asm!("vpand {out}, {a}, {b}", a = in(xmm_reg) self, b = in(xmm_reg) rhs, out = lateout(xmm_reg) out, options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(target_feature = "avx2")]
|
|
impl SimdBitwise for arch::__m256i {
|
|
#[inline(always)]
|
|
fn or(self, rhs: Self) -> Self {
|
|
unsafe {
|
|
if USE_BITWISE_INTRINSICS {
|
|
arch::_mm256_or_si256(self, rhs)
|
|
} else {
|
|
let out: _;
|
|
core::arch::asm!("vpor {out}, {a}, {b}", a = in(ymm_reg) self, b = in(ymm_reg) rhs, out = lateout(ymm_reg) out, options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}
|
|
}
|
|
|
|
#[inline(always)]
|
|
fn and(self, rhs: Self) -> Self {
|
|
unsafe {
|
|
if USE_BITWISE_INTRINSICS {
|
|
arch::_mm256_and_si256(self, rhs)
|
|
} else {
|
|
let out: _;
|
|
core::arch::asm!("vpand {out}, {a}, {b}", a = in(ymm_reg) self, b = in(ymm_reg) rhs, out = lateout(ymm_reg) out, options(pure, nomem, preserves_flags, nostack));
|
|
out
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
macro_rules! widen_128_impl {
|
|
($self:ident, $Self:ty, $reg:ident, $shift:literal, $be:literal, $offset:literal) => {unsafe {
|
|
const LEN: usize = core::mem::size_of::<$Self>();
|
|
const MASK_MOD_EQ: usize = if $be { 0 } else { $shift };
|
|
// TODO: evaluate whether there is a more efficient approach for the ignored bytes.
|
|
const INDICES: [u8; LEN] = array_op!(gen[LEN] |i| {
|
|
if (i + 1) % ($shift + 1) == MASK_MOD_EQ {
|
|
((i as u8) >> $shift) + $offset
|
|
} else {
|
|
SHUF_0
|
|
}
|
|
});
|
|
const INDICES_SIMD: $Self = unsafe { util::cast(INDICES) };
|
|
//const MASK: [u8; LEN] = array_op!(gen[LEN] |i| {
|
|
// if (i + 1) % ($shift + 1) == MASK_MOD_EQ { 0xff } else { 0x00 }
|
|
//});
|
|
//const MASK: [u8; LEN] = array_op!(gen[LEN] |i| if ((i + 1) % ($shift + 1) == $offset) { 0xff } else { 0x00 });
|
|
let out: _;
|
|
core::arch::asm!("vpshufb {out}, {in}, {indices}", in = in($reg) $self, indices = in($reg) INDICES_SIMD, out = lateout($reg) out, options(pure, nomem, preserves_flags, nostack));
|
|
if_trace_simd! {
|
|
println!("Offset: {}", $offset);
|
|
println!("Indices: {:?}", INDICES);
|
|
//println!("MASK: {MASK:?}");
|
|
}
|
|
out
|
|
//SimdBitwise::and(out, util::cast(MASK))
|
|
}};
|
|
($self:ident, $Self:ty, $reg:ident, $shift:literal, $be:ident, $offset_in:ident [$( $offset:literal ),+]) => {
|
|
match $offset_in {
|
|
$( $offset => if $be { widen_128_impl!($self, $Self, $reg, $shift, true, $offset) } else { widen_128_impl!($self, $Self, $reg, $shift, false, $offset) }, )+
|
|
_ => panic!("Unsupported widen O value"),
|
|
}
|
|
};
|
|
($self:ident, $Self:ty, $reg:ident, $shift:literal, $be:ident, $offset:ident) => {
|
|
widen_128_impl!($self, $Self, $reg, $shift, $be, $offset [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31])
|
|
};
|
|
}
|
|
|
|
macro_rules! widen_256_impl {
|
|
($self:ident, $Self:ty, $reg:ident, $shift:literal, $be:literal, $offset:literal) => {unsafe {
|
|
const LEN: usize = core::mem::size_of::<$Self>();
|
|
const MASK_MOD_EQ: usize = if $be { 0 } else { $shift };
|
|
// TODO: evaluate whether there is a more efficient approach for the ignored bytes.
|
|
const INDICES_LO: [u8; LEN] = array_op!(gen[LEN] |i| {
|
|
let i = i % LEN;
|
|
if (i + 1) % ($shift + 1) == MASK_MOD_EQ {
|
|
((i as u8) >> $shift) + $offset
|
|
} else {
|
|
SHUF_0
|
|
}
|
|
});
|
|
const INDICES_HI: [u8; LEN] = array_op!(gen[LEN] |i| {
|
|
let i = (i % (LEN / 2)) + LEN / 2;
|
|
if (i + 1) % ($shift + 1) == MASK_MOD_EQ {
|
|
((i as u8) >> $shift) + $offset
|
|
} else {
|
|
SHUF_0
|
|
}
|
|
});
|
|
const INDICES_LO_SIMD: $Self = unsafe { util::cast(INDICES_LO) };
|
|
const INDICES_HI_SIMD: $Self = unsafe { util::cast(INDICES_HI) };
|
|
const HI_MASK: [u8; LEN] = array_op!(gen[LEN] |i| if i < LEN / 2 { 0x00 } else { 0xff });
|
|
const HI_MASK_SIMD: $Self = unsafe { util::cast(HI_MASK) };
|
|
//const MASK: [u8; LEN] = array_op!(gen[LEN] |i| {
|
|
// if (i + 1) % ($shift + 1) == MASK_MOD_EQ { 0xff } else { 0x00 }
|
|
//});
|
|
//const MASK: [u8; LEN] = array_op!(gen[LEN] |i| if ((i + 1) % ($shift + 1) == $offset) { 0xff } else { 0x00 });
|
|
let lo: $Self;
|
|
core::arch::asm!("vpshufb {out:x}, {in:x}, {indices:x}", in = in($reg) $self, indices = in($reg) INDICES_LO_SIMD, out = lateout($reg) lo, options(pure, nomem, preserves_flags, nostack));
|
|
let hi: $Self;
|
|
core::arch::asm!("vpshufb {out:x}, {in:x}, {indices:x}", in = in($reg) $self, indices = in($reg) INDICES_HI_SIMD, out = lateout($reg) hi, options(pure, nomem, preserves_flags, nostack));
|
|
let out = lo.or(hi.and(HI_MASK_SIMD));
|
|
if_trace_simd! {
|
|
println!("Offset: {}", $offset);
|
|
println!("Indices: {:?},{:?}", INDICES_LO, INDICES_HI);
|
|
//println!("MASK: {MASK:?}");
|
|
}
|
|
out
|
|
//SimdBitwise::and(out, util::cast(MASK))
|
|
}};
|
|
($self:ident, $Self:ty, $reg:ident, $shift:literal, $be:ident, $offset_in:ident [$( $offset:literal ),+]) => {
|
|
match $offset_in {
|
|
$( $offset => if $be { widen_256_impl!($self, $Self, $reg, $shift, true, $offset) } else { widen_256_impl!($self, $Self, $reg, $shift, false, $offset) }, )+
|
|
_ => panic!("Unsupported widen O value"),
|
|
}
|
|
};
|
|
($self:ident, $Self:ty, $reg:ident, $shift:literal, $be:ident, $offset:ident) => {
|
|
widen_256_impl!($self, $Self, $reg, $shift, $be, $offset [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31])
|
|
};
|
|
}
|
|
|
|
#[doc(hidden)]
|
|
#[macro_export]
|
|
macro_rules! __simd__shr {
|
|
($inst:ident, $n:literal, ($in_reg:ident) $in:expr) => {{
|
|
let out: _;
|
|
core::arch::asm!(concat!(stringify!($inst), " {dst}, {src}, ", $n), src = in($in_reg) $in, dst = lateout($in_reg) out);
|
|
out
|
|
}};
|
|
(16, $n:literal, ($in_reg:ident) $in:expr) => {
|
|
$crate::simd::shr!(vpsrlw, $n, ($in_reg) $in)
|
|
};
|
|
(32, $n:literal, ($in_reg:ident) $in:expr) => {
|
|
$crate::simd::shr!(vpsrld, $n, ($in_reg) $in)
|
|
};
|
|
(64, $n:literal, ($in_reg:ident) $in:expr) => {
|
|
$crate::simd::shr!(vpsrlq, $n, ($in_reg) $in)
|
|
};
|
|
}
|
|
|
|
pub use __simd__shr as shr;
|
|
|
|
#[doc(hidden)]
|
|
#[macro_export]
|
|
macro_rules! __simd__shl {
|
|
($inst:ident, $n:literal, ($in_reg:ident) $in:expr) => {{
|
|
let out: _;
|
|
core::arch::asm!(concat!(stringify!($inst), " {dst}, {src}, ", $n), src = in($in_reg) $in, dst = lateout($in_reg) out);
|
|
out
|
|
}};
|
|
(16, $n:literal, ($in_reg:ident) $in:expr) => {
|
|
$crate::simd::shl!(vpsllw, $n, ($in_reg) $in)
|
|
};
|
|
(32, $n:literal, ($in_reg:ident) $in:expr) => {
|
|
$crate::simd::shl!(vpslld, $n, ($in_reg) $in)
|
|
};
|
|
(64, $n:literal, ($in_reg:ident) $in:expr) => {
|
|
$crate::simd::shl!(vpsllq, $n, ($in_reg) $in)
|
|
};
|
|
}
|
|
|
|
pub use __simd__shl as shl;
|
|
|
|
/*impl SimdWiden for arch::__m128i {
|
|
#[inline(always)]
|
|
fn widen<const N: usize, const O: usize, const BE: bool>(self) -> Self {
|
|
match N {
|
|
1 => self,
|
|
2 => widen_128_impl!(self, arch::__m128i, xmm_reg, 1, BE, O),
|
|
4 => widen_128_impl!(self, arch::__m128i, xmm_reg, 2, BE, O),
|
|
8 => widen_128_impl!(self, arch::__m128i, xmm_reg, 3, BE, O),
|
|
_ => panic!("Unsupported widen N value"),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl SimdWiden for arch::__m256i {
|
|
#[inline(always)]
|
|
fn widen<const N: usize, const O: usize, const BE: bool>(self) -> Self {
|
|
match N {
|
|
1 => self,
|
|
2 => widen_256_impl!(self, arch::__m256i, ymm_reg, 1, BE, O),
|
|
4 => widen_256_impl!(self, arch::__m256i, ymm_reg, 2, BE, O),
|
|
8 => widen_256_impl!(self, arch::__m256i, ymm_reg, 3, BE, O),
|
|
16 => widen_256_impl!(self, arch::__m256i, ymm_reg, 4, BE, O),
|
|
_ => panic!("Unsupported widen N value"),
|
|
}
|
|
}
|
|
}*/
|
|
|
|
#[cfg(test)]
|
|
mod test {
|
|
use crate::prelude::*;
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_splat() {
|
|
const EXPECTED: [u8; 32] = [3u8; 32];
|
|
const ACTUAL_CONST: Simd<u8, 32> = 3u8.splat::<32>();
|
|
const EQ_CONST: bool = {
|
|
let mut i = 0;
|
|
loop {
|
|
if i == 32 {
|
|
break true;
|
|
}
|
|
if ACTUAL_CONST.to_array()[i] != EXPECTED[i] {
|
|
break false;
|
|
}
|
|
i += 1;
|
|
}
|
|
};
|
|
|
|
let actual = 3u8.splat::<32>();
|
|
assert_eq!(Simd::from(actual), Simd::from_array(EXPECTED));
|
|
assert!(EQ_CONST, "constant evaluated splat did not produce the expected result");
|
|
}
|
|
|
|
#[test]
|
|
fn test_splat_zero() {
|
|
const EXPECTED: [u8; 32] = [0u8; 32];
|
|
const ACTUAL_CONST: Simd<u8, 32> = u8::splat_zero::<32>();
|
|
const EQ_CONST: bool = {
|
|
let mut i = 0;
|
|
loop {
|
|
if i == 32 {
|
|
break true;
|
|
}
|
|
if ACTUAL_CONST.to_array()[i] != EXPECTED[i] {
|
|
break false;
|
|
}
|
|
i += 1;
|
|
}
|
|
};
|
|
|
|
let actual = u8::splat_zero::<32>();
|
|
assert_eq!(Simd::from(actual), Simd::from_array(EXPECTED));
|
|
assert!(EQ_CONST, "constant evaluated splat_zero did not produce the expected result");
|
|
}
|
|
|
|
#[test]
|
|
fn test_interleave_out_128() {
|
|
const EXPECTED: [u8; 16] = array_op!(gen[16] |i| i as u8);
|
|
const A: [u8; 16] = array_op!(gen[16] |i| (i as u8) << 1);
|
|
const B: [u8; 16] = array_op!(gen[16] |i| ((i as u8) << 1) + 1);
|
|
|
|
let actual = interleave_lo_bytes_m128(Simd::from_array(A).into(), Simd::from_array(B).into());
|
|
assert_eq!(Simd::from(actual), Simd::from_array(EXPECTED));
|
|
}
|
|
|
|
#[test]
|
|
fn test_interleave_out_256() {
|
|
const EXPECTED: [u8; 32] = array_op!(gen[32] |i| i as u8);
|
|
const A: [u8; 16] = array_op!(gen[16] |i| (i as u8) << 1);
|
|
const B: [u8; 16] = array_op!(gen[16] |i| ((i as u8) << 1) + 1);
|
|
const A1: [u8; 32] = array_op!(gen[32] |i| (i as u8) << 1);
|
|
const B1: [u8; 32] = array_op!(gen[32] |i| ((i as u8) << 1) + 1);
|
|
|
|
let a = Simd::from_array(A).into();
|
|
let b = Simd::from_array(B).into();
|
|
|
|
//let a = merge_m128_m256(a, a);
|
|
//let b = merge_m128_m256(b, b);
|
|
|
|
let actual = interleave_m128(a, b);
|
|
assert_eq!(Simd::from(actual), Simd::from_array(EXPECTED));
|
|
}
|
|
|
|
#[test]
|
|
fn test_load_array() {
|
|
const ARRAY_U64_128: [u64; 2] = [46374, 5748782187];
|
|
const ARRAY_U64_256: [u64; 4] = [46374, 5748782187, 38548839, 91848923];
|
|
|
|
assert_eq!(u64::load_array(ARRAY_U64_128), Simd::from_array(ARRAY_U64_128));
|
|
assert_eq!(u64::load_array(ARRAY_U64_256), Simd::from_array(ARRAY_U64_256));
|
|
}
|
|
|
|
/*#[test]
|
|
fn test_widen() {
|
|
const SAMPLE_IN: [u8; 32] = [
|
|
1, 2, 3, 4, 5, 6, 7, 8,
|
|
1, 2, 3, 4, 5, 6, 7, 8,
|
|
9, 10, 11, 12, 13, 14, 15, 16,
|
|
9, 10, 11, 12, 13, 14, 15, 16
|
|
];
|
|
|
|
const SAMPLE_OUT_LO_BE: [u8; 32] = [
|
|
0, 1, 0, 2, 0, 3, 0, 4,
|
|
0, 5, 0, 6, 0, 7, 0, 8,
|
|
0, 1, 0, 2, 0, 3, 0, 4,
|
|
0, 5, 0, 6, 0, 7, 0, 8
|
|
];
|
|
const SAMPLE_OUT_HI_BE: [u8; 32] = [
|
|
0, 9, 0, 10, 0, 11, 0, 12,
|
|
0, 13, 0, 14, 0, 15, 0, 16,
|
|
0, 9, 0, 10, 0, 11, 0, 12,
|
|
0, 13, 0, 14, 0, 15, 0, 16
|
|
];
|
|
|
|
let sample: arch::__m256i = Simd::from_array(SAMPLE_IN).into();
|
|
let widened = sample.widen::<2, 0, true>();
|
|
//assert_eq!(Simd::from(widened), Simd::from_array(SAMPLE_OUT_LO_BE));
|
|
let widened = sample.widen::<2, 16, true>();
|
|
assert_eq!(Simd::from(widened), Simd::from_array(SAMPLE_OUT_HI_BE));
|
|
|
|
const SAMPLE_OUT_LO_LE: [u8; 32] = [
|
|
1, 0, 2, 0, 3, 0, 4, 0,
|
|
5, 0, 6, 0, 7, 0, 8, 0,
|
|
1, 0, 2, 0, 3, 0, 4, 0,
|
|
5, 0, 6, 0, 7, 0, 8, 0
|
|
];
|
|
const SAMPLE_OUT_HI_LE: [u8; 32] = [
|
|
9, 0, 10, 0, 11, 0, 12, 0,
|
|
13, 0, 14, 0, 15, 0, 16, 0,
|
|
9, 0, 10, 0, 11, 0, 12, 0,
|
|
13, 0, 14, 0, 15, 0, 16, 0
|
|
];
|
|
|
|
let sample: arch::__m256i = Simd::from_array(SAMPLE_IN).into();
|
|
let widened = sample.widen::<2, 0, false>();
|
|
//assert_eq!(Simd::from(widened), Simd::from_array(SAMPLE_OUT_LO_LE));
|
|
let widened = sample.widen::<2, 16, false>();
|
|
assert_eq!(Simd::from(widened), Simd::from_array(SAMPLE_OUT_HI_LE));
|
|
}*/
|
|
}
|