core/stdarch/crates/core_arch/src/s390x/
vector.rs

1//! s390x vector intrinsics.
2//!
3//! For more info see the [Reference Summary] or the online [IBM docs].
4//!
5//! [Reference Summary]: https://www.ibm.com/support/pages/sites/default/files/2021-05/SA22-7871-10.pdf
6//! [IBM docs]: https://www.ibm.com/docs/en/zos/2.4.0?topic=support-vector-built-in-functions
7
8#![allow(non_camel_case_types)]
9
10use crate::{core_arch::simd::*, intrinsics::simd::*, mem::MaybeUninit, mem::transmute};
11
12#[cfg(test)]
13use stdarch_test::assert_instr;
14
15use super::macros::*;
16
17types! {
18    #![unstable(feature = "stdarch_s390x", issue = "135681")]
19
20    /// s390x-specific 128-bit wide vector of sixteen packed `i8`
21    pub struct vector_signed_char(16 x i8);
22    /// s390x-specific 128-bit wide vector of sixteen packed `u8`
23    pub struct vector_unsigned_char(16 x u8);
24    /// s390x-specific 128-bit wide vector mask of sixteen packed elements
25    pub struct vector_bool_char(16 x i8);
26
27    /// s390x-specific 128-bit wide vector of eight packed `i16`
28    pub struct vector_signed_short(8 x i16);
29    /// s390x-specific 128-bit wide vector of eight packed `u16`
30    pub struct vector_unsigned_short(8 x u16);
31    /// s390x-specific 128-bit wide vector mask of eight packed elements
32    pub struct vector_bool_short(8 x i16);
33
34    /// s390x-specific 128-bit wide vector of four packed `i32`
35    pub struct vector_signed_int(4 x i32);
36    /// s390x-specific 128-bit wide vector of four packed `u32`
37    pub struct vector_unsigned_int(4 x u32);
38    /// s390x-specific 128-bit wide vector mask of four packed elements
39    pub struct vector_bool_int(4 x i32);
40
41    /// s390x-specific 128-bit wide vector of two packed `i64`
42    pub struct vector_signed_long_long(2 x i64);
43    /// s390x-specific 128-bit wide vector of two packed `u64`
44    pub struct vector_unsigned_long_long(2 x u64);
45    /// s390x-specific 128-bit wide vector mask of two packed elements
46    pub struct vector_bool_long_long(2 x i64);
47
48    /// s390x-specific 128-bit wide vector of four packed `f32`
49    pub struct vector_float(4 x f32);
50    /// s390x-specific 128-bit wide vector of two packed `f64`
51    pub struct vector_double(2 x f64);
52}
53
54#[repr(packed)]
55struct PackedTuple<T, U> {
56    x: T,
57    y: U,
58}
59
60#[allow(improper_ctypes)]
61#[rustfmt::skip]
62unsafe extern "unadjusted" {
63    #[link_name = "llvm.smax.v16i8"] fn vmxb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
64    #[link_name = "llvm.smax.v8i16"] fn vmxh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
65    #[link_name = "llvm.smax.v4i32"] fn vmxf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
66    #[link_name = "llvm.smax.v2i64"] fn vmxg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
67
68    #[link_name = "llvm.umax.v16i8"] fn vmxlb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
69    #[link_name = "llvm.umax.v8i16"] fn vmxlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
70    #[link_name = "llvm.umax.v4i32"] fn vmxlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
71    #[link_name = "llvm.umax.v2i64"] fn vmxlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
72
73    #[link_name = "llvm.smin.v16i8"] fn vmnb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
74    #[link_name = "llvm.smin.v8i16"] fn vmnh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
75    #[link_name = "llvm.smin.v4i32"] fn vmnf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
76    #[link_name = "llvm.smin.v2i64"] fn vmng(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
77
78    #[link_name = "llvm.umin.v16i8"] fn vmnlb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
79    #[link_name = "llvm.umin.v8i16"] fn vmnlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
80    #[link_name = "llvm.umin.v4i32"] fn vmnlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
81    #[link_name = "llvm.umin.v2i64"] fn vmnlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
82
83    #[link_name = "llvm.nearbyint.v4f32"] fn nearbyint_v4f32(a: vector_float) -> vector_float;
84    #[link_name = "llvm.nearbyint.v2f64"] fn nearbyint_v2f64(a: vector_double) -> vector_double;
85
86    #[link_name = "llvm.rint.v4f32"] fn rint_v4f32(a: vector_float) -> vector_float;
87    #[link_name = "llvm.rint.v2f64"] fn rint_v2f64(a: vector_double) -> vector_double;
88
89    #[link_name = "llvm.roundeven.v4f32"] fn roundeven_v4f32(a: vector_float) -> vector_float;
90    #[link_name = "llvm.roundeven.v2f64"] fn roundeven_v2f64(a: vector_double) -> vector_double;
91
92    #[link_name = "llvm.s390.vsra"] fn vsra(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
93    #[link_name = "llvm.s390.vsrl"] fn vsrl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
94    #[link_name = "llvm.s390.vsl"] fn vsl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
95
96    #[link_name = "llvm.s390.vsrab"] fn vsrab(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
97    #[link_name = "llvm.s390.vsrlb"] fn vsrlb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
98    #[link_name = "llvm.s390.vslb"] fn vslb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
99
100    #[link_name = "llvm.s390.vsldb"] fn vsldb(a: i8x16, b: i8x16, c: u32) -> i8x16;
101    #[link_name = "llvm.s390.vsld"] fn vsld(a: i8x16, b: i8x16, c: u32) -> i8x16;
102    #[link_name = "llvm.s390.vsrd"] fn vsrd(a: i8x16, b: i8x16, c: u32) -> i8x16;
103
104    #[link_name = "llvm.fshl.v16i8"] fn fshlb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
105    #[link_name = "llvm.fshl.v8i16"] fn fshlh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
106    #[link_name = "llvm.fshl.v4i32"] fn fshlf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
107    #[link_name = "llvm.fshl.v2i64"] fn fshlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
108
109    #[link_name = "llvm.s390.verimb"] fn verimb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char, d: i32) -> vector_signed_char;
110    #[link_name = "llvm.s390.verimh"] fn verimh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short, d: i32) -> vector_signed_short;
111    #[link_name = "llvm.s390.verimf"] fn verimf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int, d: i32) -> vector_signed_int;
112    #[link_name = "llvm.s390.verimg"] fn verimg(a: vector_signed_long_long, b: vector_signed_long_long, c: vector_signed_long_long, d: i32) -> vector_signed_long_long;
113
114    #[link_name = "llvm.s390.vperm"] fn vperm(a: vector_signed_char, b: vector_signed_char, c: vector_unsigned_char) -> vector_signed_char;
115
116    #[link_name = "llvm.s390.vsumb"] fn vsumb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int;
117    #[link_name = "llvm.s390.vsumh"] fn vsumh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
118
119    #[link_name = "llvm.s390.vsumgh"] fn vsumgh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long;
120    #[link_name = "llvm.s390.vsumgf"] fn vsumgf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
121
122    #[link_name = "llvm.s390.vsumqf"] fn vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128;
123    #[link_name = "llvm.s390.vsumqg"] fn vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
124
125    #[link_name = "llvm.s390.vscbiq"] fn vscbiq(a: u128, b: u128) -> u128;
126    #[link_name = "llvm.s390.vsbiq"] fn vsbiq(a: u128, b: u128, c: u128) -> u128;
127    #[link_name = "llvm.s390.vsbcbiq"] fn vsbcbiq(a: u128, b: u128, c: u128) -> u128;
128
129    #[link_name = "llvm.s390.vacq"] fn vacq(a: u128, b: u128, c: u128) -> u128;
130
131    #[link_name = "llvm.s390.vscbib"] fn vscbib(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
132    #[link_name = "llvm.s390.vscbih"] fn vscbih(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
133    #[link_name = "llvm.s390.vscbif"] fn vscbif(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
134    #[link_name = "llvm.s390.vscbig"] fn vscbig(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
135
136    #[link_name = "llvm.s390.vfaeb"] fn vfaeb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
137    #[link_name = "llvm.s390.vfaeh"] fn vfaeh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
138    #[link_name = "llvm.s390.vfaef"] fn vfaef(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
139
140    #[link_name = "llvm.s390.vfaezb"] fn vfaezb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
141    #[link_name = "llvm.s390.vfaezh"] fn vfaezh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
142    #[link_name = "llvm.s390.vfaezf"] fn vfaezf(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
143
144    #[link_name = "llvm.s390.vfaebs"] fn vfaebs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
145    #[link_name = "llvm.s390.vfaehs"] fn vfaehs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
146    #[link_name = "llvm.s390.vfaefs"] fn vfaefs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
147
148    #[link_name = "llvm.s390.vfaezbs"] fn vfaezbs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
149    #[link_name = "llvm.s390.vfaezhs"] fn vfaezhs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
150    #[link_name = "llvm.s390.vfaezfs"] fn vfaezfs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
151
152    #[link_name = "llvm.s390.vll"] fn vll(a: u32, b: *const u8) -> vector_signed_char;
153    #[link_name = "llvm.s390.vstl"] fn vstl(a: vector_signed_char, b: u32, c: *mut u8);
154
155    #[link_name = "llvm.s390.vlrl"] fn vlrl(a: u32, b: *const u8) -> vector_unsigned_char;
156    #[link_name = "llvm.s390.vstrl"] fn vstrl(a: vector_unsigned_char, b: u32, c: *mut u8);
157
158    #[link_name = "llvm.s390.lcbb"] fn lcbb(a: *const u8, b: u32) -> u32;
159    #[link_name = "llvm.s390.vlbb"] fn vlbb(a: *const u8, b: u32) -> MaybeUninit<vector_signed_char>;
160
161    #[link_name = "llvm.s390.vpksh"] fn vpksh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_char;
162    #[link_name = "llvm.s390.vpksf"] fn vpksf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_short;
163    #[link_name = "llvm.s390.vpksg"] fn vpksg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_int;
164
165    #[link_name = "llvm.s390.vpklsh"] fn vpklsh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char;
166    #[link_name = "llvm.s390.vpklsf"] fn vpklsf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short;
167    #[link_name = "llvm.s390.vpklsg"] fn vpklsg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_unsigned_int;
168
169    #[link_name = "llvm.s390.vpkshs"] fn vpkshs(a: vector_signed_short, b: vector_signed_short) -> PackedTuple<vector_signed_char, i32>;
170    #[link_name = "llvm.s390.vpksfs"] fn vpksfs(a: vector_signed_int, b: vector_signed_int) -> PackedTuple<vector_signed_short, i32>;
171    #[link_name = "llvm.s390.vpksgs"] fn vpksgs(a: vector_signed_long_long, b: vector_signed_long_long) -> PackedTuple<vector_signed_int, i32>;
172
173    #[link_name = "llvm.s390.vpklshs"] fn vpklshs(a: vector_unsigned_short, b: vector_unsigned_short) -> PackedTuple<vector_unsigned_char, i32>;
174    #[link_name = "llvm.s390.vpklsfs"] fn vpklsfs(a: vector_unsigned_int, b: vector_unsigned_int) -> PackedTuple<vector_unsigned_short, i32>;
175    #[link_name = "llvm.s390.vpklsgs"] fn vpklsgs(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> PackedTuple<vector_unsigned_int, i32>;
176
177    #[link_name = "llvm.s390.vuplbw"] fn vuplbw (a: vector_signed_char) -> vector_signed_short;
178    #[link_name = "llvm.s390.vuplhw"] fn vuplhw (a: vector_signed_short) -> vector_signed_int;
179    #[link_name = "llvm.s390.vuplfw"] fn vuplfw (a: vector_signed_int) -> vector_signed_long_long;
180    #[link_name = "llvm.s390.vupllb"] fn vupllb (a: vector_unsigned_char) -> vector_unsigned_short;
181    #[link_name = "llvm.s390.vupllh"] fn vupllh (a: vector_unsigned_short) -> vector_unsigned_int;
182    #[link_name = "llvm.s390.vupllf"] fn vupllf (a: vector_unsigned_int) -> vector_unsigned_long_long;
183
184    #[link_name = "llvm.s390.vavgb"] fn vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
185    #[link_name = "llvm.s390.vavgh"] fn vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
186    #[link_name = "llvm.s390.vavgf"] fn vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
187    #[link_name = "llvm.s390.vavgg"] fn vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
188
189    #[link_name = "llvm.s390.vavglb"] fn vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
190    #[link_name = "llvm.s390.vavglh"] fn vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
191    #[link_name = "llvm.s390.vavglf"] fn vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
192    #[link_name = "llvm.s390.vavglg"] fn vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
193
194    #[link_name = "llvm.s390.vcksm"] fn vcksm(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
195
196    #[link_name = "llvm.s390.vmeb"] fn vmeb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short;
197    #[link_name = "llvm.s390.vmeh"] fn vmeh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int;
198    #[link_name = "llvm.s390.vmef"] fn vmef(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long;
199
200    #[link_name = "llvm.s390.vmleb"] fn vmleb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
201    #[link_name = "llvm.s390.vmleh"] fn vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
202    #[link_name = "llvm.s390.vmlef"] fn vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
203
204    #[link_name = "llvm.s390.vmob"] fn vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short;
205    #[link_name = "llvm.s390.vmoh"] fn vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int;
206    #[link_name = "llvm.s390.vmof"] fn vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long;
207
208    #[link_name = "llvm.s390.vmlob"] fn vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
209    #[link_name = "llvm.s390.vmloh"] fn vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
210    #[link_name = "llvm.s390.vmlof"] fn vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
211
212    #[link_name = "llvm.s390.vmhb"] fn vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
213    #[link_name = "llvm.s390.vmhh"] fn vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
214    #[link_name = "llvm.s390.vmhf"] fn vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
215
216    #[link_name = "llvm.s390.vmlhb"] fn vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
217    #[link_name = "llvm.s390.vmlhh"] fn vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
218    #[link_name = "llvm.s390.vmlhf"] fn vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
219
220    #[link_name = "llvm.s390.vmaeb"] fn vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
221    #[link_name = "llvm.s390.vmaeh"] fn vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
222    #[link_name = "llvm.s390.vmaef"] fn vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
223
224    #[link_name = "llvm.s390.vmaleb"] fn vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
225    #[link_name = "llvm.s390.vmaleh"] fn vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
226    #[link_name = "llvm.s390.vmalef"] fn vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
227
228    #[link_name = "llvm.s390.vmaob"] fn vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
229    #[link_name = "llvm.s390.vmaoh"] fn vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
230    #[link_name = "llvm.s390.vmaof"] fn vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
231
232    #[link_name = "llvm.s390.vmalob"] fn vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
233    #[link_name = "llvm.s390.vmaloh"] fn vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
234    #[link_name = "llvm.s390.vmalof"] fn vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
235
236    #[link_name = "llvm.s390.vmahb"] fn vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
237    #[link_name = "llvm.s390.vmahh"] fn vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
238    #[link_name = "llvm.s390.vmahf"] fn vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
239
240    #[link_name = "llvm.s390.vmalhb"] fn vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
241    #[link_name = "llvm.s390.vmalhh"] fn vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
242    #[link_name = "llvm.s390.vmalhf"] fn vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
243
244    #[link_name = "llvm.s390.vmalb"] fn vmalb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
245    #[link_name = "llvm.s390.vmalh"] fn vmalh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
246    #[link_name = "llvm.s390.vmalf"] fn vmalf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
247
248    #[link_name = "llvm.s390.vmallb"] fn vmallb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
249    #[link_name = "llvm.s390.vmallh"] fn vmallh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
250    #[link_name = "llvm.s390.vmallf"] fn vmallf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
251
252    #[link_name = "llvm.s390.vgfmb"] fn vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
253    #[link_name = "llvm.s390.vgfmh"] fn vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
254    #[link_name = "llvm.s390.vgfmf"] fn vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
255    #[link_name = "llvm.s390.vgfmg"] fn vgfmg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
256
257    #[link_name = "llvm.s390.vgfmab"] fn vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
258    #[link_name = "llvm.s390.vgfmah"] fn vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
259    #[link_name = "llvm.s390.vgfmaf"] fn vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
260    #[link_name = "llvm.s390.vgfmag"] fn vgfmag(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128) -> u128;
261
262    #[link_name = "llvm.s390.vbperm"] fn vbperm(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_long_long;
263
264    #[link_name = "llvm.s390.vftcisb"] fn vftcisb(a: vector_float, b: u32) -> PackedTuple<vector_bool_int, i32>;
265    #[link_name = "llvm.s390.vftcidb"] fn vftcidb(a: vector_double, b: u32) -> PackedTuple<vector_bool_long_long, i32>;
266
267    #[link_name = "llvm.s390.vtm"] fn vtm(a: i8x16, b: i8x16) -> i32;
268
269    #[link_name = "llvm.s390.vstrsb"] fn vstrsb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
270    #[link_name = "llvm.s390.vstrsh"] fn vstrsh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
271    #[link_name = "llvm.s390.vstrsf"] fn vstrsf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
272
273    #[link_name = "llvm.s390.vstrszb"] fn vstrszb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
274    #[link_name = "llvm.s390.vstrszh"] fn vstrszh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
275    #[link_name = "llvm.s390.vstrszf"] fn vstrszf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
276
277    #[link_name = "llvm.s390.vistrb"] fn vistrb(a: vector_unsigned_char) -> vector_unsigned_char;
278    #[link_name = "llvm.s390.vistrh"] fn vistrh(a: vector_unsigned_short) -> vector_unsigned_short;
279    #[link_name = "llvm.s390.vistrf"] fn vistrf(a: vector_unsigned_int) -> vector_unsigned_int;
280
281    #[link_name = "llvm.s390.vistrbs"] fn vistrbs(a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
282    #[link_name = "llvm.s390.vistrhs"] fn vistrhs(a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32>;
283    #[link_name = "llvm.s390.vistrfs"] fn vistrfs(a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32>;
284
285    #[link_name = "llvm.s390.vmslg"] fn vmslg(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128, d: u32) -> u128;
286
287    #[link_name = "llvm.s390.vstrcb"] fn vstrcb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
288    #[link_name = "llvm.s390.vstrch"] fn vstrch(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
289    #[link_name = "llvm.s390.vstrcf"] fn vstrcf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
290
291    #[link_name = "llvm.s390.vstrcbs"] fn vstrcbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
292    #[link_name = "llvm.s390.vstrchs"] fn vstrchs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
293    #[link_name = "llvm.s390.vstrcfs"] fn vstrcfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
294
295    #[link_name = "llvm.s390.vstrczb"] fn vstrczb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
296    #[link_name = "llvm.s390.vstrczh"] fn vstrczh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
297    #[link_name = "llvm.s390.vstrczf"] fn vstrczf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
298
299    #[link_name = "llvm.s390.vstrczbs"] fn vstrczbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
300    #[link_name = "llvm.s390.vstrczhs"] fn vstrczhs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
301    #[link_name = "llvm.s390.vstrczfs"] fn vstrczfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
302
303    #[link_name = "llvm.s390.vfeeb"] fn vfeeb(a: i8x16, b: i8x16) -> i8x16;
304    #[link_name = "llvm.s390.vfeeh"] fn vfeeh(a: i16x8, b: i16x8) -> i16x8;
305    #[link_name = "llvm.s390.vfeef"] fn vfeef(a: i32x4, b: i32x4) -> i32x4;
306
307    #[link_name = "llvm.s390.vfeezb"] fn vfeezb(a: i8x16, b: i8x16) -> i8x16;
308    #[link_name = "llvm.s390.vfeezh"] fn vfeezh(a: i16x8, b: i16x8) -> i16x8;
309    #[link_name = "llvm.s390.vfeezf"] fn vfeezf(a: i32x4, b: i32x4) -> i32x4;
310
311    #[link_name = "llvm.s390.vfeebs"] fn vfeebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
312    #[link_name = "llvm.s390.vfeehs"] fn vfeehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
313    #[link_name = "llvm.s390.vfeefs"] fn vfeefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
314
315    #[link_name = "llvm.s390.vfeezbs"] fn vfeezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
316    #[link_name = "llvm.s390.vfeezhs"] fn vfeezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
317    #[link_name = "llvm.s390.vfeezfs"] fn vfeezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
318
319    #[link_name = "llvm.s390.vfeneb"] fn vfeneb(a: i8x16, b: i8x16) -> i8x16;
320    #[link_name = "llvm.s390.vfeneh"] fn vfeneh(a: i16x8, b: i16x8) -> i16x8;
321    #[link_name = "llvm.s390.vfenef"] fn vfenef(a: i32x4, b: i32x4) -> i32x4;
322
323    #[link_name = "llvm.s390.vfenezb"] fn vfenezb(a: i8x16, b: i8x16) -> i8x16;
324    #[link_name = "llvm.s390.vfenezh"] fn vfenezh(a: i16x8, b: i16x8) -> i16x8;
325    #[link_name = "llvm.s390.vfenezf"] fn vfenezf(a: i32x4, b: i32x4) -> i32x4;
326
327    #[link_name = "llvm.s390.vfenebs"] fn vfenebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
328    #[link_name = "llvm.s390.vfenehs"] fn vfenehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
329    #[link_name = "llvm.s390.vfenefs"] fn vfenefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
330
331    #[link_name = "llvm.s390.vfenezbs"] fn vfenezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
332    #[link_name = "llvm.s390.vfenezhs"] fn vfenezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
333    #[link_name = "llvm.s390.vfenezfs"] fn vfenezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
334}
335
336impl_from! { i8x16, u8x16,  i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 }
337
338impl_neg! { i8x16 : 0 }
339impl_neg! { i16x8 : 0 }
340impl_neg! { i32x4 : 0 }
341impl_neg! { i64x2 : 0 }
342impl_neg! { f32x4 : 0f32 }
343impl_neg! { f64x2 : 0f64 }
344
345#[repr(simd)]
346struct ShuffleMask<const N: usize>([u32; N]);
347
348impl<const N: usize> ShuffleMask<N> {
349    const fn reverse() -> Self {
350        let mut index = [0; N];
351        let mut i = 0;
352        while i < N {
353            index[i] = (N - i - 1) as u32;
354            i += 1;
355        }
356        ShuffleMask(index)
357    }
358
359    const fn merge_low() -> Self {
360        let mut mask = [0; N];
361        let mut i = N / 2;
362        let mut index = 0;
363        while index < N {
364            mask[index] = i as u32;
365            mask[index + 1] = (i + N) as u32;
366
367            i += 1;
368            index += 2;
369        }
370        ShuffleMask(mask)
371    }
372
373    const fn merge_high() -> Self {
374        let mut mask = [0; N];
375        let mut i = 0;
376        let mut index = 0;
377        while index < N {
378            mask[index] = i as u32;
379            mask[index + 1] = (i + N) as u32;
380
381            i += 1;
382            index += 2;
383        }
384        ShuffleMask(mask)
385    }
386
387    const fn pack() -> Self {
388        let mut mask = [0; N];
389        let mut i = 1;
390        let mut index = 0;
391        while index < N {
392            mask[index] = i as u32;
393
394            i += 2;
395            index += 1;
396        }
397        ShuffleMask(mask)
398    }
399
400    const fn unpack_low() -> Self {
401        let mut mask = [0; N];
402        let mut i = 0;
403        while i < N {
404            mask[i] = (N + i) as u32;
405            i += 1;
406        }
407        ShuffleMask(mask)
408    }
409
410    const fn unpack_high() -> Self {
411        let mut mask = [0; N];
412        let mut i = 0;
413        while i < N {
414            mask[i] = i as u32;
415            i += 1;
416        }
417        ShuffleMask(mask)
418    }
419}
420
421const fn genmask<const MASK: u16>() -> [u8; 16] {
422    let mut bits = MASK;
423    let mut elements = [0u8; 16];
424
425    let mut i = 0;
426    while i < 16 {
427        elements[i] = match bits & (1u16 << 15) {
428            0 => 0,
429            _ => 0xFF,
430        };
431
432        bits <<= 1;
433        i += 1;
434    }
435
436    elements
437}
438
439const fn genmasks(bit_width: u32, a: u8, b: u8) -> u64 {
440    let bit_width = bit_width as u8;
441    let a = a % bit_width;
442    let mut b = b % bit_width;
443    if a > b {
444        b = bit_width - 1;
445    }
446
447    // of course these indices start from the left
448    let a = (bit_width - 1) - a;
449    let b = (bit_width - 1) - b;
450
451    ((1u64.wrapping_shl(a as u32 + 1)) - 1) & !((1u64.wrapping_shl(b as u32)) - 1)
452}
453
454const fn validate_block_boundary(block_boundary: u16) -> u32 {
455    assert!(
456        block_boundary.is_power_of_two() && block_boundary >= 64 && block_boundary <= 4096,
457        "block boundary must be a constant power of 2 from 64 to 4096",
458    );
459
460    // so that 64 is encoded as 0, 128 as 1, ect.
461    block_boundary as u32 >> 7
462}
463
464enum FindImm {
465    Eq = 4,
466    Ne = 12,
467    EqIdx = 0,
468    NeIdx = 8,
469}
470
471#[macro_use]
472mod sealed {
473    use super::*;
474
475    #[unstable(feature = "stdarch_s390x", issue = "135681")]
476    pub trait VectorAdd<Other> {
477        type Result;
478        unsafe fn vec_add(self, other: Other) -> Self::Result;
479    }
480
481    macro_rules! impl_add {
482        ($name:ident, $a:ty, $instr:ident) => {
483            impl_add!($name, $a, $a, $a, $instr);
484        };
485        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
486            #[inline]
487            #[target_feature(enable = "vector")]
488            #[cfg_attr(test, assert_instr($instr))]
489            pub unsafe fn $name(a: $a, b: $b) -> $c {
490                transmute(simd_add(transmute(a), b))
491            }
492
493            #[unstable(feature = "stdarch_s390x", issue = "135681")]
494            impl VectorAdd<$b> for $a {
495                type Result = $c;
496
497                #[inline]
498                #[target_feature(enable = "vector")]
499                unsafe fn vec_add(self, other: $b) -> Self::Result {
500                    $name(self, other)
501                }
502            }
503        };
504    }
505
506    #[rustfmt::skip]
507    mod impl_add {
508        use super::*;
509
510        impl_add!(va_sc, vector_signed_char, vab);
511        impl_add!(va_uc, vector_unsigned_char, vab);
512        impl_add!(va_sh, vector_signed_short, vah);
513        impl_add!(va_uh, vector_unsigned_short, vah);
514        impl_add!(va_sf, vector_signed_int, vaf);
515        impl_add!(va_uf, vector_unsigned_int, vaf);
516        impl_add!(va_sg, vector_signed_long_long, vag);
517        impl_add!(va_ug, vector_unsigned_long_long, vag);
518
519        impl_add!(va_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vab);
520        impl_add!(va_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vab);
521        impl_add!(va_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vah);
522        impl_add!(va_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vah);
523        impl_add!(va_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vaf);
524        impl_add!(va_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vaf);
525        impl_add!(va_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vag);
526        impl_add!(va_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vag);
527
528        impl_add!(va_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vab);
529        impl_add!(va_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vab);
530        impl_add!(va_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vah);
531        impl_add!(va_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vah);
532        impl_add!(va_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vaf);
533        impl_add!(va_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vaf);
534        impl_add!(va_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vag);
535        impl_add!(va_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vag);
536
537        impl_add!(va_double, vector_double, vfadb);
538
539        #[inline]
540        #[target_feature(enable = "vector")]
541        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfasb))]
542        pub unsafe fn va_float(a: vector_float, b: vector_float) -> vector_float {
543            transmute(simd_add(a, b))
544        }
545
546        #[unstable(feature = "stdarch_s390x", issue = "135681")]
547        impl VectorAdd<Self> for vector_float {
548            type Result = Self;
549
550            #[inline]
551            #[target_feature(enable = "vector")]
552            unsafe fn vec_add(self, other: Self) -> Self::Result {
553                va_float(self, other)
554            }
555        }
556    }
557
558    #[unstable(feature = "stdarch_s390x", issue = "135681")]
559    pub trait VectorSub<Other> {
560        type Result;
561        unsafe fn vec_sub(self, other: Other) -> Self::Result;
562    }
563
564    macro_rules! impl_sub {
565        ($name:ident, $a:ty, $instr:ident) => {
566            impl_sub!($name, $a, $a, $a, $instr);
567        };
568        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
569            #[inline]
570            #[target_feature(enable = "vector")]
571            #[cfg_attr(test, assert_instr($instr))]
572            pub unsafe fn $name(a: $a, b: $b) -> $c {
573                transmute(simd_sub(transmute(a), b))
574            }
575
576            #[unstable(feature = "stdarch_s390x", issue = "135681")]
577            impl VectorSub<$b> for $a {
578                type Result = $c;
579
580                #[inline]
581                #[target_feature(enable = "vector")]
582                unsafe fn vec_sub(self, other: $b) -> Self::Result {
583                    $name(self, other)
584                }
585            }
586        };
587    }
588
589    #[rustfmt::skip]
590    mod impl_sub {
591        use super::*;
592
593        impl_sub!(vs_sc, vector_signed_char, vsb);
594        impl_sub!(vs_uc, vector_unsigned_char, vsb);
595        impl_sub!(vs_sh, vector_signed_short, vsh);
596        impl_sub!(vs_uh, vector_unsigned_short, vsh);
597        impl_sub!(vs_sf, vector_signed_int, vsf);
598        impl_sub!(vs_uf, vector_unsigned_int, vsf);
599        impl_sub!(vs_sg, vector_signed_long_long, vsg);
600        impl_sub!(vs_ug, vector_unsigned_long_long, vsg);
601
602        impl_sub!(vs_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vsb);
603        impl_sub!(vs_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vsb);
604        impl_sub!(vs_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vsh);
605        impl_sub!(vs_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vsh);
606        impl_sub!(vs_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vsf);
607        impl_sub!(vs_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vsf);
608        impl_sub!(vs_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vsg);
609        impl_sub!(vs_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vsg);
610
611        impl_sub!(vs_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vsb);
612        impl_sub!(vs_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vsb);
613        impl_sub!(vs_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vsh);
614        impl_sub!(vs_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vsh);
615        impl_sub!(vs_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vsf);
616        impl_sub!(vs_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vsf);
617        impl_sub!(vs_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vsg);
618        impl_sub!(vs_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vsg);
619
620        impl_sub!(vs_double, vector_double, vfsdb);
621
622        #[inline]
623        #[target_feature(enable = "vector")]
624        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfssb))]
625        pub unsafe fn vs_float(a: vector_float, b: vector_float) -> vector_float {
626            transmute(simd_sub(a, b))
627        }
628
629        #[unstable(feature = "stdarch_s390x", issue = "135681")]
630        impl VectorSub<Self> for vector_float {
631            type Result = Self;
632
633            #[inline]
634            #[target_feature(enable = "vector")]
635            unsafe fn vec_sub(self, other: Self) -> Self::Result {
636                vs_float(self, other)
637            }
638        }
639    }
640
641    #[unstable(feature = "stdarch_s390x", issue = "135681")]
642    pub trait VectorMul {
643        unsafe fn vec_mul(self, b: Self) -> Self;
644    }
645
646    macro_rules! impl_mul {
647        ($name:ident, $a:ty, std_simd) => {
648            #[unstable(feature = "stdarch_s390x", issue = "135681")]
649            impl VectorMul for $a {
650                #[inline]
651                #[target_feature(enable = "vector")]
652                unsafe fn vec_mul(self, other: Self) -> Self {
653                    transmute(simd_mul(transmute(self), other))
654                }
655            }
656        };
657        ($name:ident, $a:ty, $instr:ident) => {
658            #[inline]
659            #[target_feature(enable = "vector")]
660            #[cfg_attr(test, assert_instr($instr))]
661            pub unsafe fn $name(a: $a, b: $a) -> $a {
662                transmute(simd_mul(transmute(a), b))
663            }
664
665            #[unstable(feature = "stdarch_s390x", issue = "135681")]
666            impl VectorMul for $a {
667                #[inline]
668                #[target_feature(enable = "vector")]
669                unsafe fn vec_mul(self, other: Self) -> Self {
670                    $name(self, other)
671                }
672            }
673        };
674    }
675
676    #[rustfmt::skip]
677    mod impl_mul {
678        use super::*;
679
680        impl_mul!(vml_sc, vector_signed_char, vmlb);
681        impl_mul!(vml_uc, vector_unsigned_char, vmlb);
682        impl_mul!(vml_sh, vector_signed_short, vmlhw);
683        impl_mul!(vml_uh, vector_unsigned_short, vmlhw);
684        impl_mul!(vml_sf, vector_signed_int, vmlf);
685        impl_mul!(vml_uf, vector_unsigned_int, vmlf);
686        impl_mul!(vml_sg, vector_signed_long_long, std_simd);
687        impl_mul!(vml_ug, vector_unsigned_long_long, std_simd);
688
689        impl_mul!(vml_float, vector_float, std_simd);
690        impl_mul!(vml_double, vector_double, vfmdb);
691    }
692
693    #[unstable(feature = "stdarch_s390x", issue = "135681")]
694    pub trait VectorMax<Other> {
695        type Result;
696        unsafe fn vec_max(self, b: Other) -> Self::Result;
697    }
698
699    test_impl! { vec_vmxsb (a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [vmxb, vmxb] }
700    test_impl! { vec_vmxsh (a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [vmxh, vmxh] }
701    test_impl! { vec_vmxsf (a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [vmxf, vmxf] }
702    test_impl! { vec_vmxsg (a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [vmxg, vmxg] }
703
704    test_impl! { vec_vmxslb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vmxlb, vmxlb] }
705    test_impl! { vec_vmxslh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vmxlh, vmxlh] }
706    test_impl! { vec_vmxslf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vmxlf, vmxlf] }
707    test_impl! { vec_vmxslg (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vmxlg, vmxlg] }
708
709    impl_vec_trait! { [VectorMax vec_max] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) }
710
711    test_impl! { vec_vfmaxsb (a: vector_float, b: vector_float) -> vector_float [simd_fmax, "vector-enhancements-1" vfmaxsb ] }
712    test_impl! { vec_vfmaxdb (a: vector_double, b: vector_double) -> vector_double [simd_fmax, "vector-enhancements-1" vfmaxdb] }
713
714    impl_vec_trait!([VectorMax vec_max] vec_vfmaxsb (vector_float, vector_float) -> vector_float);
715    impl_vec_trait!([VectorMax vec_max] vec_vfmaxdb (vector_double, vector_double) -> vector_double);
716
717    #[unstable(feature = "stdarch_s390x", issue = "135681")]
718    pub trait VectorMin<Other> {
719        type Result;
720        unsafe fn vec_min(self, b: Other) -> Self::Result;
721    }
722
723    test_impl! { vec_vmnsb (a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [vmnb, vmnb] }
724    test_impl! { vec_vmnsh (a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [vmnh, vmnh] }
725    test_impl! { vec_vmnsf (a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [vmnf, vmnf] }
726    test_impl! { vec_vmnsg (a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [vmng, vmng] }
727
728    test_impl! { vec_vmnslb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vmnlb, vmnlb] }
729    test_impl! { vec_vmnslh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vmnlh, vmnlh] }
730    test_impl! { vec_vmnslf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vmnlf, vmnlf] }
731    test_impl! { vec_vmnslg (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vmnlg, vmnlg] }
732
733    impl_vec_trait! { [VectorMin vec_min] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) }
734
735    test_impl! { vec_vfminsb (a: vector_float, b: vector_float) -> vector_float [simd_fmin, "vector-enhancements-1" vfminsb]  }
736    test_impl! { vec_vfmindb (a: vector_double, b: vector_double) -> vector_double [simd_fmin, "vector-enhancements-1" vfmindb]  }
737
738    impl_vec_trait!([VectorMin vec_min] vec_vfminsb (vector_float, vector_float) -> vector_float);
739    impl_vec_trait!([VectorMin vec_min] vec_vfmindb (vector_double, vector_double) -> vector_double);
740
741    #[unstable(feature = "stdarch_s390x", issue = "135681")]
742    pub trait VectorAbs {
743        unsafe fn vec_abs(self) -> Self;
744    }
745
746    macro_rules! impl_abs {
747        ($name:ident, $ty:ident) => {
748            #[inline]
749            #[target_feature(enable = "vector")]
750            unsafe fn $name(v: s_t_l!($ty)) -> s_t_l!($ty) {
751                v.vec_max(-v)
752            }
753
754            impl_vec_trait! { [VectorAbs vec_abs] $name (s_t_l!($ty)) }
755        };
756    }
757
758    impl_abs! { vec_abs_i8, i8x16 }
759    impl_abs! { vec_abs_i16, i16x8 }
760    impl_abs! { vec_abs_i32, i32x4 }
761    impl_abs! { vec_abs_i64, i64x2 }
762
763    test_impl! { vec_abs_f32 (v: vector_float) -> vector_float [ simd_fabs, "vector-enhancements-1" vflpsb ] }
764    test_impl! { vec_abs_f64 (v: vector_double) -> vector_double [ simd_fabs, vflpdb ] }
765
766    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f32 (vector_float) }
767    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f64 (vector_double) }
768
769    #[unstable(feature = "stdarch_s390x", issue = "135681")]
770    pub trait VectorNabs {
771        unsafe fn vec_nabs(self) -> Self;
772    }
773
774    #[inline]
775    #[target_feature(enable = "vector")]
776    #[cfg_attr(
777        all(test, target_feature = "vector-enhancements-1"),
778        assert_instr(vflnsb)
779    )]
780    unsafe fn vec_nabs_f32(a: vector_float) -> vector_float {
781        simd_neg(simd_fabs(a))
782    }
783
784    #[inline]
785    #[target_feature(enable = "vector")]
786    #[cfg_attr(test, assert_instr(vflndb))]
787    unsafe fn vec_nabs_f64(a: vector_double) -> vector_double {
788        simd_neg(simd_fabs(a))
789    }
790
791    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f32 (vector_float) }
792    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f64 (vector_double) }
793
794    #[unstable(feature = "stdarch_s390x", issue = "135681")]
795    pub trait VectorNmsub {
796        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self;
797    }
798
799    #[inline]
800    #[target_feature(enable = "vector")]
801    #[cfg_attr(
802        all(test, target_feature = "vector-enhancements-2"),
803        assert_instr(vfnmssb)
804    )]
805    unsafe fn vec_nmsub_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
806        simd_neg(simd_fma(a, b, simd_neg(c)))
807    }
808
809    #[unstable(feature = "stdarch_s390x", issue = "135681")]
810    impl VectorNmsub for vector_float {
811        #[target_feature(enable = "vector")]
812        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
813            vec_nmsub_f32(self, b, c)
814        }
815    }
816
817    #[inline]
818    #[target_feature(enable = "vector")]
819    #[cfg_attr(
820        all(test, target_feature = "vector-enhancements-2"),
821        assert_instr(vfnmsdb)
822    )]
823    unsafe fn vec_nmsub_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
824        simd_neg(simd_fma(a, b, simd_neg(c)))
825    }
826
827    #[unstable(feature = "stdarch_s390x", issue = "135681")]
828    impl VectorNmsub for vector_double {
829        #[target_feature(enable = "vector")]
830        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
831            vec_nmsub_f64(self, b, c)
832        }
833    }
834
835    #[unstable(feature = "stdarch_s390x", issue = "135681")]
836    pub trait VectorNmadd {
837        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self;
838    }
839
840    #[inline]
841    #[target_feature(enable = "vector")]
842    #[cfg_attr(
843        all(test, target_feature = "vector-enhancements-2"),
844        assert_instr(vfnmasb)
845    )]
846    unsafe fn vec_nmadd_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
847        simd_neg(simd_fma(a, b, c))
848    }
849
850    #[unstable(feature = "stdarch_s390x", issue = "135681")]
851    impl VectorNmadd for vector_float {
852        #[target_feature(enable = "vector")]
853        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
854            vec_nmadd_f32(self, b, c)
855        }
856    }
857
858    #[inline]
859    #[target_feature(enable = "vector")]
860    #[cfg_attr(
861        all(test, target_feature = "vector-enhancements-2"),
862        assert_instr(vfnmadb)
863    )]
864    unsafe fn vec_nmadd_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
865        simd_neg(simd_fma(a, b, c))
866    }
867
868    #[unstable(feature = "stdarch_s390x", issue = "135681")]
869    impl VectorNmadd for vector_double {
870        #[target_feature(enable = "vector")]
871        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
872            vec_nmadd_f64(self, b, c)
873        }
874    }
875
876    #[unstable(feature = "stdarch_s390x", issue = "135681")]
877    pub trait VectorSplat {
878        unsafe fn vec_splat<const IMM: u32>(self) -> Self;
879    }
880
881    #[inline]
882    #[target_feature(enable = "vector")]
883    #[cfg_attr(test, assert_instr(vrepb, IMM2 = 1))]
884    unsafe fn vrepb<const IMM2: u32>(a: vector_signed_char) -> vector_signed_char {
885        static_assert_uimm_bits!(IMM2, 4);
886        simd_shuffle(a, a, const { u32x16::from_array([IMM2; 16]) })
887    }
888
889    #[inline]
890    #[target_feature(enable = "vector")]
891    #[cfg_attr(test, assert_instr(vreph, IMM2 = 1))]
892    unsafe fn vreph<const IMM2: u32>(a: vector_signed_short) -> vector_signed_short {
893        static_assert_uimm_bits!(IMM2, 3);
894        simd_shuffle(a, a, const { u32x8::from_array([IMM2; 8]) })
895    }
896
897    #[inline]
898    #[target_feature(enable = "vector")]
899    #[cfg_attr(test, assert_instr(vrepf, IMM2 = 1))]
900    unsafe fn vrepf<const IMM2: u32>(a: vector_signed_int) -> vector_signed_int {
901        static_assert_uimm_bits!(IMM2, 2);
902        simd_shuffle(a, a, const { u32x4::from_array([IMM2; 4]) })
903    }
904
905    #[inline]
906    #[target_feature(enable = "vector")]
907    #[cfg_attr(test, assert_instr(vrepg, IMM2 = 1))]
908    unsafe fn vrepg<const IMM2: u32>(a: vector_signed_long_long) -> vector_signed_long_long {
909        static_assert_uimm_bits!(IMM2, 1);
910        simd_shuffle(a, a, const { u32x2::from_array([IMM2; 2]) })
911    }
912
913    macro_rules! impl_vec_splat {
914        ($ty:ty, $fun:ident) => {
915            #[unstable(feature = "stdarch_s390x", issue = "135681")]
916            impl VectorSplat for $ty {
917                #[inline]
918                #[target_feature(enable = "vector")]
919                unsafe fn vec_splat<const IMM: u32>(self) -> Self {
920                    transmute($fun::<IMM>(transmute(self)))
921                }
922            }
923        };
924    }
925
926    impl_vec_splat! { vector_signed_char, vrepb }
927    impl_vec_splat! { vector_unsigned_char, vrepb }
928    impl_vec_splat! { vector_bool_char, vrepb }
929    impl_vec_splat! { vector_signed_short, vreph }
930    impl_vec_splat! { vector_unsigned_short, vreph }
931    impl_vec_splat! { vector_bool_short, vreph }
932    impl_vec_splat! { vector_signed_int, vrepf }
933    impl_vec_splat! { vector_unsigned_int, vrepf }
934    impl_vec_splat! { vector_bool_int, vrepf }
935    impl_vec_splat! { vector_signed_long_long, vrepg }
936    impl_vec_splat! { vector_unsigned_long_long, vrepg }
937    impl_vec_splat! { vector_bool_long_long, vrepg }
938
939    impl_vec_splat! { vector_float, vrepf }
940    impl_vec_splat! { vector_double, vrepg }
941
942    #[unstable(feature = "stdarch_s390x", issue = "135681")]
943    pub trait VectorSplats<Output> {
944        unsafe fn vec_splats(self) -> Output;
945    }
946
947    macro_rules! impl_vec_splats {
948        ($(($fn:ident ($ty:ty, $shortty:tt) $instr:ident)),*) => {
949            $(
950                #[inline]
951                #[target_feature(enable = "vector")]
952                #[cfg_attr(test, assert_instr($instr))]
953                pub unsafe fn $fn(v: $ty) -> s_t_l!($shortty) {
954                    transmute($shortty::splat(v))
955                }
956
957                #[unstable(feature = "stdarch_s390x", issue = "135681")]
958                impl VectorSplats<s_t_l!($shortty)> for $ty {
959                    #[inline]
960                    #[target_feature(enable = "vector")]
961                    unsafe fn vec_splats(self) -> s_t_l!($shortty) {
962                        $fn (self)
963                    }
964                }
965            )*
966        }
967    }
968
969    impl_vec_splats! {
970        (vec_splats_u8 (u8, u8x16) vrepb),
971        (vec_splats_i8 (i8, i8x16) vrepb),
972        (vec_splats_u16 (u16, u16x8) vreph),
973        (vec_splats_i16 (i16, i16x8) vreph),
974        (vec_splats_u32 (u32, u32x4) vrepf),
975        (vec_splats_i32 (i32, i32x4) vrepf),
976        (vec_splats_u64 (u64, u64x2) vlvgp),
977        (vec_splats_i64 (i64, i64x2) vlvgp),
978        (vec_splats_f32 (f32, f32x4) vrepf),
979        (vec_splats_f64 (f64, f64x2) vrepg)
980    }
981
982    macro_rules! impl_bool_vec_splats {
983        ($(($ty:ty, $shortty:tt, $boolty:ty)),*) => {
984            $(
985                #[unstable(feature = "stdarch_s390x", issue = "135681")]
986                impl VectorSplats<$boolty> for $ty {
987                    #[inline]
988                    #[target_feature(enable = "vector")]
989                    unsafe fn vec_splats(self) -> $boolty {
990                        transmute($shortty::splat(self))
991                    }
992                }
993            )*
994        }
995    }
996
997    impl_bool_vec_splats! {
998        (u8, u8x16, vector_bool_char),
999        (i8, i8x16, vector_bool_char),
1000        (u16, u16x8, vector_bool_short),
1001        (i16, i16x8, vector_bool_short),
1002        (u32, u32x4, vector_bool_int),
1003        (i32, i32x4, vector_bool_int),
1004        (u64, u64x2, vector_bool_long_long),
1005        (i64, i64x2, vector_bool_long_long)
1006    }
1007
1008    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1009    pub trait CountBits {
1010        type Result;
1011
1012        unsafe fn vec_cntlz(self) -> Self::Result;
1013        unsafe fn vec_cnttz(self) -> Self::Result;
1014        unsafe fn vec_popcnt(self) -> Self::Result;
1015    }
1016
1017    macro_rules! impl_count_bits {
1018        ($ty:tt) => {
1019            #[unstable(feature = "stdarch_s390x", issue = "135681")]
1020            impl CountBits for $ty {
1021                type Result = t_u!($ty);
1022
1023                #[inline]
1024                #[target_feature(enable = "vector")]
1025                unsafe fn vec_cntlz(self) -> Self::Result {
1026                    transmute(simd_ctlz(self))
1027                }
1028
1029                #[inline]
1030                #[target_feature(enable = "vector")]
1031                unsafe fn vec_cnttz(self) -> Self::Result {
1032                    transmute(simd_cttz(self))
1033                }
1034
1035                #[inline]
1036                #[target_feature(enable = "vector")]
1037                unsafe fn vec_popcnt(self) -> Self::Result {
1038                    transmute(simd_ctpop(self))
1039                }
1040            }
1041        };
1042    }
1043
1044    impl_count_bits!(vector_signed_char);
1045    impl_count_bits!(vector_unsigned_char);
1046    impl_count_bits!(vector_signed_short);
1047    impl_count_bits!(vector_unsigned_short);
1048    impl_count_bits!(vector_signed_int);
1049    impl_count_bits!(vector_unsigned_int);
1050    impl_count_bits!(vector_signed_long_long);
1051    impl_count_bits!(vector_unsigned_long_long);
1052
1053    test_impl! { vec_clzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1054    test_impl! { vec_clzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1055    test_impl! { vec_clzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1056    test_impl! { vec_clzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1057
1058    test_impl! { vec_clzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1059    test_impl! { vec_clzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1060    test_impl! { vec_clzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1061    test_impl! { vec_clzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1062
1063    test_impl! { vec_ctzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1064    test_impl! { vec_ctzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1065    test_impl! { vec_ctzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1066    test_impl! { vec_ctzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1067
1068    test_impl! { vec_ctzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1069    test_impl! { vec_ctzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1070    test_impl! { vec_ctzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1071    test_impl! { vec_ctzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1072
1073    test_impl! { vec_vpopctb_signed +(a: vector_signed_char) -> vector_signed_char [simd_ctpop, vpopctb] }
1074    test_impl! { vec_vpopcth_signed +(a: vector_signed_short) -> vector_signed_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1075    test_impl! { vec_vpopctf_signed +(a: vector_signed_int) -> vector_signed_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1076    test_impl! { vec_vpopctg_signed +(a: vector_signed_long_long) -> vector_signed_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1077
1078    test_impl! { vec_vpopctb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctpop, vpopctb] }
1079    test_impl! { vec_vpopcth_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1080    test_impl! { vec_vpopctf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1081    test_impl! { vec_vpopctg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1082
1083    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1084    pub trait VectorAnd<Other> {
1085        type Result;
1086        unsafe fn vec_and(self, b: Other) -> Self::Result;
1087    }
1088
1089    impl_vec_trait! { [VectorAnd vec_and] ~(simd_and) }
1090
1091    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1092    pub trait VectorOr<Other> {
1093        type Result;
1094        unsafe fn vec_or(self, b: Other) -> Self::Result;
1095    }
1096
1097    impl_vec_trait! { [VectorOr vec_or] ~(simd_or) }
1098
1099    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1100    pub trait VectorXor<Other> {
1101        type Result;
1102        unsafe fn vec_xor(self, b: Other) -> Self::Result;
1103    }
1104
1105    impl_vec_trait! { [VectorXor vec_xor] ~(simd_xor) }
1106
1107    #[inline]
1108    #[target_feature(enable = "vector")]
1109    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vno))]
1110    unsafe fn nor(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1111        let a: u8x16 = transmute(a);
1112        let b: u8x16 = transmute(b);
1113        transmute(simd_xor(simd_or(a, b), u8x16::splat(0xff)))
1114    }
1115
1116    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1117    pub trait VectorNor<Other> {
1118        type Result;
1119        unsafe fn vec_nor(self, b: Other) -> Self::Result;
1120    }
1121
1122    impl_vec_trait! { [VectorNor vec_nor]+ 2c (nor) }
1123
1124    #[inline]
1125    #[target_feature(enable = "vector")]
1126    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnn))]
1127    unsafe fn nand(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1128        let a: u8x16 = transmute(a);
1129        let b: u8x16 = transmute(b);
1130        transmute(simd_xor(simd_and(a, b), u8x16::splat(0xff)))
1131    }
1132
1133    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1134    pub trait VectorNand<Other> {
1135        type Result;
1136        unsafe fn vec_nand(self, b: Other) -> Self::Result;
1137    }
1138
1139    impl_vec_trait! { [VectorNand vec_nand]+ 2c (nand) }
1140
1141    #[inline]
1142    #[target_feature(enable = "vector")]
1143    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnx))]
1144    unsafe fn eqv(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1145        let a: u8x16 = transmute(a);
1146        let b: u8x16 = transmute(b);
1147        transmute(simd_xor(simd_xor(a, b), u8x16::splat(0xff)))
1148    }
1149
1150    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1151    pub trait VectorEqv<Other> {
1152        type Result;
1153        unsafe fn vec_eqv(self, b: Other) -> Self::Result;
1154    }
1155
1156    impl_vec_trait! { [VectorEqv vec_eqv]+ 2c (eqv) }
1157
1158    #[inline]
1159    #[target_feature(enable = "vector")]
1160    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnc))]
1161    unsafe fn andc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1162        let a = transmute(a);
1163        let b = transmute(b);
1164        transmute(simd_and(simd_xor(u8x16::splat(0xff), b), a))
1165    }
1166
1167    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1168    pub trait VectorAndc<Other> {
1169        type Result;
1170        unsafe fn vec_andc(self, b: Other) -> Self::Result;
1171    }
1172
1173    impl_vec_trait! { [VectorAndc vec_andc]+ 2c (andc) }
1174
1175    #[inline]
1176    #[target_feature(enable = "vector")]
1177    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(voc))]
1178    unsafe fn orc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1179        let a = transmute(a);
1180        let b = transmute(b);
1181        transmute(simd_or(simd_xor(u8x16::splat(0xff), b), a))
1182    }
1183
1184    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1185    pub trait VectorOrc<Other> {
1186        type Result;
1187        unsafe fn vec_orc(self, b: Other) -> Self::Result;
1188    }
1189
1190    impl_vec_trait! { [VectorOrc vec_orc]+ 2c (orc) }
1191
1192    test_impl! { vec_roundc_f32 (a: vector_float) -> vector_float [nearbyint_v4f32,  "vector-enhancements-1" vfisb] }
1193    test_impl! { vec_roundc_f64 (a: vector_double) -> vector_double [nearbyint_v2f64, vfidb] }
1194
1195    // FIXME(llvm) llvm trunk already lowers roundeven to vfidb, but rust does not use it yet
1196    // use https://godbolt.org/z/cWq95fexe to check, and enable the instruction test when it works
1197    test_impl! { vec_round_f32 (a: vector_float) -> vector_float [roundeven_v4f32, _] }
1198    test_impl! { vec_round_f64 (a: vector_double) -> vector_double [roundeven_v2f64, _] }
1199
1200    test_impl! { vec_rint_f32 (a: vector_float) -> vector_float [rint_v4f32, "vector-enhancements-1" vfisb] }
1201    test_impl! { vec_rint_f64 (a: vector_double) -> vector_double [rint_v2f64, vfidb] }
1202
1203    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1204    pub trait VectorRoundc {
1205        unsafe fn vec_roundc(self) -> Self;
1206    }
1207
1208    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1209    pub trait VectorRound {
1210        unsafe fn vec_round(self) -> Self;
1211    }
1212
1213    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1214    pub trait VectorRint {
1215        unsafe fn vec_rint(self) -> Self;
1216    }
1217
1218    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f32 (vector_float) }
1219    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f64 (vector_double) }
1220
1221    impl_vec_trait! { [VectorRound vec_round] vec_round_f32 (vector_float) }
1222    impl_vec_trait! { [VectorRound vec_round] vec_round_f64 (vector_double) }
1223
1224    impl_vec_trait! { [VectorRint vec_rint] vec_rint_f32 (vector_float) }
1225    impl_vec_trait! { [VectorRint vec_rint] vec_rint_f64 (vector_double) }
1226
1227    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1228    pub trait VectorTrunc {
1229        // same as vec_roundz
1230        unsafe fn vec_trunc(self) -> Self;
1231    }
1232
1233    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1234    pub trait VectorCeil {
1235        // same as vec_roundp
1236        unsafe fn vec_ceil(self) -> Self;
1237    }
1238
1239    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1240    pub trait VectorFloor {
1241        // same as vec_roundm
1242        unsafe fn vec_floor(self) -> Self;
1243    }
1244
1245    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_float) }
1246    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_double) }
1247
1248    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_float) }
1249    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_double) }
1250
1251    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_float) }
1252    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_double) }
1253
1254    macro_rules! impl_vec_shift {
1255        ([$Trait:ident $m:ident] ($b:ident, $h:ident, $w:ident, $g:ident)) => {
1256            impl_vec_trait!{ [$Trait $m]+ $b (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1257            impl_vec_trait!{ [$Trait $m]+ $b (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1258            impl_vec_trait!{ [$Trait $m]+ $h (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1259            impl_vec_trait!{ [$Trait $m]+ $h (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1260            impl_vec_trait!{ [$Trait $m]+ $w (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1261            impl_vec_trait!{ [$Trait $m]+ $w (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1262            impl_vec_trait!{ [$Trait $m]+ $g (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1263            impl_vec_trait!{ [$Trait $m]+ $g (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1264        };
1265    }
1266
1267    macro_rules! impl_shift {
1268        ($fun:ident $intr:ident $ty:ident) => {
1269            #[inline]
1270            #[target_feature(enable = "vector")]
1271            #[cfg_attr(test, assert_instr($fun))]
1272            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1273                let a = transmute(a);
1274                // use the remainder of b by the width of a's elements to prevent UB
1275                let b = simd_rem(transmute(b), <t_t_s!($ty)>::splat($ty::BITS as $ty));
1276
1277                transmute($intr(a, b))
1278            }
1279        };
1280    }
1281
1282    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1283    pub trait VectorSl<Other> {
1284        type Result;
1285        unsafe fn vec_sl(self, b: Other) -> Self::Result;
1286    }
1287
1288    impl_shift! { veslvb simd_shl u8 }
1289    impl_shift! { veslvh simd_shl u16 }
1290    impl_shift! { veslvf simd_shl u32 }
1291    impl_shift! { veslvg simd_shl u64 }
1292
1293    impl_vec_shift! { [VectorSl vec_sl] (veslvb, veslvh, veslvf, veslvg) }
1294
1295    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1296    pub trait VectorSr<Other> {
1297        type Result;
1298        unsafe fn vec_sr(self, b: Other) -> Self::Result;
1299    }
1300
1301    impl_shift! { vesrlvb simd_shr u8 }
1302    impl_shift! { vesrlvh simd_shr u16 }
1303    impl_shift! { vesrlvf simd_shr u32 }
1304    impl_shift! { vesrlvg simd_shr u64 }
1305
1306    impl_vec_shift! { [VectorSr vec_sr] (vesrlvb, vesrlvh, vesrlvf, vesrlvg) }
1307
1308    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1309    pub trait VectorSra<Other> {
1310        type Result;
1311        unsafe fn vec_sra(self, b: Other) -> Self::Result;
1312    }
1313
1314    impl_shift! { vesravb simd_shr i8 }
1315    impl_shift! { vesravh simd_shr i16 }
1316    impl_shift! { vesravf simd_shr i32 }
1317    impl_shift! { vesravg simd_shr i64 }
1318
1319    impl_vec_shift! { [VectorSra vec_sra] (vesravb, vesravh, vesravf, vesravg) }
1320
1321    macro_rules! impl_vec_shift_byte {
1322        ([$trait:ident $m:ident] ($f:ident)) => {
1323            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_signed_char) -> vector_unsigned_char }
1324            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1325            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_signed_char) -> vector_signed_char }
1326            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1327            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_signed_short) -> vector_unsigned_short }
1328            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1329            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_signed_short) -> vector_signed_short }
1330            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1331            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_signed_int) -> vector_unsigned_int }
1332            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1333            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_signed_int) -> vector_signed_int }
1334            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1335            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_signed_long_long) -> vector_unsigned_long_long }
1336            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1337            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_signed_long_long) -> vector_signed_long_long }
1338            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1339            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_signed_int) -> vector_float }
1340            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_unsigned_int) -> vector_float }
1341            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_signed_long_long) -> vector_double }
1342            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_unsigned_long_long) -> vector_double }
1343        };
1344    }
1345
1346    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1347    pub trait VectorSlb<Other> {
1348        type Result;
1349        unsafe fn vec_slb(self, b: Other) -> Self::Result;
1350    }
1351
1352    impl_vec_shift_byte! { [VectorSlb vec_slb] (vslb) }
1353
1354    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1355    pub trait VectorSrab<Other> {
1356        type Result;
1357        unsafe fn vec_srab(self, b: Other) -> Self::Result;
1358    }
1359
1360    impl_vec_shift_byte! { [VectorSrab vec_srab] (vsrab) }
1361
1362    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1363    pub trait VectorSrb<Other> {
1364        type Result;
1365        unsafe fn vec_srb(self, b: Other) -> Self::Result;
1366    }
1367
1368    impl_vec_shift_byte! { [VectorSrb vec_srb] (vsrlb) }
1369
1370    macro_rules! impl_vec_shift_long {
1371        ([$trait:ident $m:ident] ($f:ident)) => {
1372            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1373            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1374            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_char) -> vector_unsigned_short }
1375            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_char) -> vector_signed_short }
1376            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_char) -> vector_unsigned_int }
1377            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_char) -> vector_signed_int }
1378            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_char) -> vector_unsigned_long_long }
1379            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_char) -> vector_signed_long_long }
1380        };
1381    }
1382
1383    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1384    pub trait VectorSrl<Other> {
1385        type Result;
1386        unsafe fn vec_srl(self, b: Other) -> Self::Result;
1387    }
1388
1389    impl_vec_shift_long! { [VectorSrl vec_srl] (vsrl) }
1390
1391    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1392    pub trait VectorSral<Other> {
1393        type Result;
1394        unsafe fn vec_sral(self, b: Other) -> Self::Result;
1395    }
1396
1397    impl_vec_shift_long! { [VectorSral vec_sral] (vsra) }
1398
1399    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1400    pub trait VectorSll<Other> {
1401        type Result;
1402        unsafe fn vec_sll(self, b: Other) -> Self::Result;
1403    }
1404
1405    impl_vec_shift_long! { [VectorSll vec_sll] (vsl) }
1406
1407    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1408    pub trait VectorRl<Other> {
1409        type Result;
1410        unsafe fn vec_rl(self, b: Other) -> Self::Result;
1411    }
1412
1413    macro_rules! impl_rot {
1414        ($fun:ident $intr:ident $ty:ident) => {
1415            #[inline]
1416            #[target_feature(enable = "vector")]
1417            #[cfg_attr(test, assert_instr($fun))]
1418            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1419                transmute($intr(transmute(a), transmute(a), transmute(b)))
1420            }
1421        };
1422    }
1423
1424    impl_rot! { verllvb fshlb u8 }
1425    impl_rot! { verllvh fshlh u16 }
1426    impl_rot! { verllvf fshlf u32 }
1427    impl_rot! { verllvg fshlg u64 }
1428
1429    impl_vec_shift! { [VectorRl vec_rl] (verllvb, verllvh, verllvf, verllvg) }
1430
1431    macro_rules! test_rot_imm {
1432        ($fun:ident $instr:ident $intr:ident $ty:ident) => {
1433            #[inline]
1434            #[target_feature(enable = "vector")]
1435            #[cfg_attr(test, assert_instr($instr))]
1436            unsafe fn $fun(a: t_t_l!($ty), bits: core::ffi::c_ulong) -> t_t_l!($ty) {
1437                // mod by the number of bits in a's element type to prevent UB
1438                let bits = (bits % $ty::BITS as core::ffi::c_ulong) as $ty;
1439                let a = transmute(a);
1440                let b = <t_t_s!($ty)>::splat(bits);
1441
1442                transmute($intr(a, a, transmute(b)))
1443            }
1444        };
1445    }
1446
1447    test_rot_imm! { verllvb_imm verllb fshlb u8 }
1448    test_rot_imm! { verllvh_imm verllh fshlh u16 }
1449    test_rot_imm! { verllvf_imm verllf fshlf u32 }
1450    test_rot_imm! { verllvg_imm verllg fshlg u64 }
1451
1452    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1453    pub trait VectorRli {
1454        unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self;
1455    }
1456
1457    macro_rules! impl_rot_imm {
1458        ($($ty:ident, $intr:ident),*) => {
1459            $(
1460                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1461                impl VectorRli for $ty {
1462                    #[inline]
1463                    #[target_feature(enable = "vector")]
1464                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1465                        transmute($intr(transmute(self), bits))
1466                    }
1467                }
1468
1469                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1470                impl VectorRli for t_u!($ty) {
1471                    #[inline]
1472                    #[target_feature(enable = "vector")]
1473                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1474                        $intr(self, bits)
1475                    }
1476                }
1477            )*
1478        }
1479    }
1480
1481    impl_rot_imm! {
1482        vector_signed_char, verllvb_imm,
1483        vector_signed_short, verllvh_imm,
1484        vector_signed_int, verllvf_imm,
1485        vector_signed_long_long, verllvg_imm
1486    }
1487
1488    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1489    pub trait VectorRlMask<Other> {
1490        unsafe fn vec_rl_mask<const IMM8: u8>(self, other: Other) -> Self;
1491    }
1492
1493    macro_rules! impl_rl_mask {
1494        ($($ty:ident, $intr:ident, $fun:ident),*) => {
1495            $(
1496                #[inline]
1497                #[target_feature(enable = "vector")]
1498                #[cfg_attr(test, assert_instr($intr, IMM8 = 6))]
1499                unsafe fn $fun<const IMM8: u8>(a: $ty, b: t_u!($ty)) -> $ty {
1500                    // mod by the number of bits in a's element type to prevent UB
1501                    $intr(a, a, transmute(b), const { (IMM8 % <l_t_t!($ty)>::BITS as u8) as i32 })
1502                }
1503
1504                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1505                impl VectorRlMask<t_u!($ty)> for $ty {
1506                    #[inline]
1507                    #[target_feature(enable = "vector")]
1508                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1509                        $fun::<IMM8>(self, other)
1510                    }
1511                }
1512
1513                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1514                impl VectorRlMask<t_u!($ty)> for t_u!($ty) {
1515                    #[inline]
1516                    #[target_feature(enable = "vector")]
1517                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1518                        transmute($fun::<IMM8>(transmute(self), transmute(other)))
1519                    }
1520                }
1521            )*
1522        }
1523    }
1524
1525    impl_rl_mask! {
1526        vector_signed_char, verimb, test_verimb,
1527        vector_signed_short, verimh, test_verimh,
1528        vector_signed_int, verimf, test_verimf,
1529        vector_signed_long_long, verimg, test_verimg
1530    }
1531
1532    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1533    pub trait VectorReve {
1534        unsafe fn vec_reve(self) -> Self;
1535    }
1536
1537    #[repr(simd)]
1538    struct ReverseMask<const N: usize>([u32; N]);
1539
1540    impl<const N: usize> ReverseMask<N> {
1541        const fn new() -> Self {
1542            let mut index = [0; N];
1543            let mut i = 0;
1544            while i < N {
1545                index[i] = (N - i - 1) as u32;
1546                i += 1;
1547            }
1548            ReverseMask(index)
1549        }
1550    }
1551
1552    macro_rules! impl_reve {
1553        ($($ty:ident, $fun:ident, $instr:ident),*) => {
1554            $(
1555                #[inline]
1556                #[target_feature(enable = "vector")]
1557                #[cfg_attr(test, assert_instr($instr))]
1558                unsafe fn $fun(a: $ty) -> $ty {
1559                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1560                    simd_shuffle(a, a, const { ShuffleMask::<N>::reverse() })
1561                }
1562
1563                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1564                impl VectorReve for $ty {
1565                    #[inline]
1566                    #[target_feature(enable = "vector")]
1567                    unsafe fn vec_reve(self) -> Self {
1568                        $fun(self)
1569                    }
1570                }
1571
1572                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1573                impl VectorReve for t_u!($ty) {
1574                    #[inline]
1575                    #[target_feature(enable = "vector")]
1576                    unsafe fn vec_reve(self) -> Self {
1577                        transmute($fun(transmute(self)))
1578                    }
1579                }
1580
1581                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1582                impl VectorReve for t_b!($ty) {
1583                    #[inline]
1584                    #[target_feature(enable = "vector")]
1585                    unsafe fn vec_reve(self) -> Self {
1586                        transmute($fun(transmute(self)))
1587                    }
1588                }
1589            )*
1590        }
1591    }
1592
1593    impl_reve! {
1594        vector_signed_char, reveb, vperm,
1595        vector_signed_short, reveh, vperm,
1596        vector_signed_int, revef, vperm,
1597        vector_signed_long_long, reveg, vpdi
1598    }
1599
1600    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1601    impl VectorReve for vector_float {
1602        #[inline]
1603        #[target_feature(enable = "vector")]
1604        unsafe fn vec_reve(self) -> Self {
1605            transmute(transmute::<_, vector_signed_int>(self).vec_reve())
1606        }
1607    }
1608
1609    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1610    impl VectorReve for vector_double {
1611        #[inline]
1612        #[target_feature(enable = "vector")]
1613        unsafe fn vec_reve(self) -> Self {
1614            transmute(transmute::<_, vector_signed_long_long>(self).vec_reve())
1615        }
1616    }
1617
1618    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1619    pub trait VectorRevb {
1620        unsafe fn vec_revb(self) -> Self;
1621    }
1622
1623    test_impl! { bswapb (a: vector_signed_char) -> vector_signed_char [simd_bswap, _] }
1624    test_impl! { bswaph (a: vector_signed_short) -> vector_signed_short [simd_bswap, vperm] }
1625    test_impl! { bswapf (a: vector_signed_int) -> vector_signed_int [simd_bswap, vperm] }
1626    test_impl! { bswapg (a: vector_signed_long_long) -> vector_signed_long_long [simd_bswap, vperm] }
1627
1628    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_unsigned_char) }
1629    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_signed_char) }
1630    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_unsigned_short) }
1631    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_signed_short) }
1632    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_unsigned_int) }
1633    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_signed_int) }
1634    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_unsigned_long_long) }
1635    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_signed_long_long) }
1636
1637    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1638    impl VectorRevb for vector_float {
1639        #[inline]
1640        #[target_feature(enable = "vector")]
1641        unsafe fn vec_revb(self) -> Self {
1642            transmute(transmute::<_, vector_signed_int>(self).vec_revb())
1643        }
1644    }
1645
1646    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1647    impl VectorRevb for vector_double {
1648        #[inline]
1649        #[target_feature(enable = "vector")]
1650        unsafe fn vec_revb(self) -> Self {
1651            transmute(transmute::<_, vector_signed_long_long>(self).vec_revb())
1652        }
1653    }
1654
1655    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1656    pub trait VectorMergel {
1657        unsafe fn vec_mergel(self, other: Self) -> Self;
1658    }
1659
1660    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1661    pub trait VectorMergeh {
1662        unsafe fn vec_mergeh(self, other: Self) -> Self;
1663    }
1664
1665    macro_rules! impl_merge {
1666        ($($ty:ident, $mergel:ident, $mergeh:ident),*) => {
1667            $(
1668                #[inline]
1669                #[target_feature(enable = "vector")]
1670                #[cfg_attr(test, assert_instr($mergel))]
1671                unsafe fn $mergel(a: $ty, b: $ty) -> $ty {
1672                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1673                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_low() })
1674                }
1675
1676                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1677                impl VectorMergel for $ty {
1678                    #[inline]
1679                    #[target_feature(enable = "vector")]
1680                    unsafe fn vec_mergel(self, other: Self) -> Self {
1681                        $mergel(self, other)
1682                    }
1683                }
1684
1685                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1686                impl VectorMergel for t_u!($ty) {
1687                    #[inline]
1688                    #[target_feature(enable = "vector")]
1689                    unsafe fn vec_mergel(self, other: Self) -> Self {
1690                        transmute($mergel(transmute(self), transmute(other)))
1691                    }
1692                }
1693
1694                #[inline]
1695                #[target_feature(enable = "vector")]
1696                #[cfg_attr(test, assert_instr($mergeh))]
1697                unsafe fn $mergeh(a: $ty, b: $ty) -> $ty {
1698                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1699                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_high() })
1700                }
1701
1702                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1703                impl VectorMergeh for $ty {
1704                    #[inline]
1705                    #[target_feature(enable = "vector")]
1706                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1707                        $mergeh(self, other)
1708                    }
1709                }
1710
1711                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1712                impl VectorMergeh for t_u!($ty) {
1713                    #[inline]
1714                    #[target_feature(enable = "vector")]
1715                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1716                        transmute($mergeh(transmute(self), transmute(other)))
1717                    }
1718                }
1719            )*
1720        }
1721    }
1722
1723    impl_merge! {
1724        vector_signed_char, vmrlb, vmrhb,
1725        vector_signed_short, vmrlh, vmrhh,
1726        vector_signed_int, vmrlf, vmrhf,
1727        vector_signed_long_long, vmrlg, vmrhg
1728    }
1729
1730    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1731    pub trait VectorPerm {
1732        unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self;
1733    }
1734
1735    macro_rules! impl_merge {
1736        ($($ty:ident),*) => {
1737            $(
1738                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1739                impl VectorPerm for $ty {
1740                    #[inline]
1741                    #[target_feature(enable = "vector")]
1742                    unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self {
1743                        transmute(vperm(transmute(self), transmute(other), c))
1744                    }
1745                }
1746            )*
1747        }
1748    }
1749
1750    impl_merge! {
1751        vector_signed_char,
1752        vector_signed_short,
1753        vector_signed_int,
1754        vector_signed_long_long,
1755        vector_unsigned_char,
1756        vector_unsigned_short,
1757        vector_unsigned_int,
1758        vector_unsigned_long_long,
1759        vector_bool_char,
1760        vector_bool_short,
1761        vector_bool_int,
1762        vector_bool_long_long,
1763        vector_float,
1764        vector_double
1765    }
1766
1767    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1768    pub trait VectorSumU128 {
1769        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char;
1770    }
1771
1772    #[inline]
1773    #[target_feature(enable = "vector")]
1774    #[cfg_attr(test, assert_instr(vsumqf))]
1775    pub unsafe fn vec_vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128 {
1776        transmute(vsumqf(a, b))
1777    }
1778
1779    #[inline]
1780    #[target_feature(enable = "vector")]
1781    #[cfg_attr(test, assert_instr(vsumqg))]
1782    pub unsafe fn vec_vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128 {
1783        transmute(vsumqg(a, b))
1784    }
1785
1786    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1787    impl VectorSumU128 for vector_unsigned_int {
1788        #[inline]
1789        #[target_feature(enable = "vector")]
1790        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1791            transmute(vec_vsumqf(self, other))
1792        }
1793    }
1794
1795    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1796    impl VectorSumU128 for vector_unsigned_long_long {
1797        #[inline]
1798        #[target_feature(enable = "vector")]
1799        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1800            transmute(vec_vsumqg(self, other))
1801        }
1802    }
1803
1804    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1805    pub trait VectorSum2 {
1806        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long;
1807    }
1808
1809    test_impl! { vec_vsumgh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long [vsumgh, vsumgh] }
1810    test_impl! { vec_vsumgf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [vsumgf, vsumgf] }
1811
1812    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1813    impl VectorSum2 for vector_unsigned_short {
1814        #[inline]
1815        #[target_feature(enable = "vector")]
1816        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1817            vec_vsumgh(self, other)
1818        }
1819    }
1820
1821    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1822    impl VectorSum2 for vector_unsigned_int {
1823        #[inline]
1824        #[target_feature(enable = "vector")]
1825        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1826            vec_vsumgf(self, other)
1827        }
1828    }
1829
1830    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1831    pub trait VectorSum4 {
1832        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int;
1833    }
1834
1835    test_impl! { vec_vsumb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int [vsumb, vsumb] }
1836    test_impl! { vec_vsumh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int [vsumh, vsumh] }
1837
1838    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1839    impl VectorSum4 for vector_unsigned_char {
1840        #[inline]
1841        #[target_feature(enable = "vector")]
1842        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1843            vec_vsumb(self, other)
1844        }
1845    }
1846
1847    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1848    impl VectorSum4 for vector_unsigned_short {
1849        #[inline]
1850        #[target_feature(enable = "vector")]
1851        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1852            vec_vsumh(self, other)
1853        }
1854    }
1855
1856    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1857    pub trait VectorSubc<Other> {
1858        type Result;
1859        unsafe fn vec_subc(self, b: Other) -> Self::Result;
1860    }
1861
1862    test_impl! { vec_vscbib (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vscbib, vscbib] }
1863    test_impl! { vec_vscbih (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vscbih, vscbih] }
1864    test_impl! { vec_vscbif (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vscbif, vscbif] }
1865    test_impl! { vec_vscbig (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vscbig, vscbig] }
1866
1867    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbib (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1868    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbih (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1869    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbif (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1870    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbig (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1871
1872    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1873    pub trait VectorSqrt {
1874        unsafe fn vec_sqrt(self) -> Self;
1875    }
1876
1877    test_impl! { vec_sqrt_f32 (v: vector_float) -> vector_float [ simd_fsqrt, "vector-enhancements-1" vfsqsb ] }
1878    test_impl! { vec_sqrt_f64 (v: vector_double) -> vector_double [ simd_fsqrt, vfsqdb ] }
1879
1880    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f32 (vector_float) }
1881    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f64 (vector_double) }
1882
1883    macro_rules! vfae_wrapper {
1884        ($($name:ident $ty:ident)*) => {
1885            $(
1886                #[inline]
1887                #[target_feature(enable = "vector")]
1888                #[cfg_attr(test, assert_instr($name, IMM = 0))]
1889                unsafe fn $name<const IMM: i32>(
1890                    a: $ty,
1891                    b: $ty,
1892                ) -> $ty {
1893                    super::$name(a, b, IMM)
1894                }
1895            )*
1896        }
1897     }
1898
1899    vfae_wrapper! {
1900       vfaeb vector_signed_char
1901       vfaeh vector_signed_short
1902       vfaef vector_signed_int
1903
1904       vfaezb vector_signed_char
1905       vfaezh vector_signed_short
1906       vfaezf vector_signed_int
1907    }
1908
1909    macro_rules! impl_vfae {
1910        ([idx_cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1911            impl_vfae! { [idx_cc $Trait $m] $imm
1912                $b vector_signed_char vector_signed_char
1913                $b vector_unsigned_char vector_unsigned_char
1914                $b vector_bool_char vector_unsigned_char
1915
1916                $h vector_signed_short vector_signed_short
1917                $h vector_unsigned_short vector_unsigned_short
1918                $h vector_bool_short vector_unsigned_short
1919
1920                $f vector_signed_int vector_signed_int
1921                $f vector_unsigned_int vector_unsigned_int
1922                $f vector_bool_int vector_unsigned_int
1923            }
1924        };
1925        ([idx_cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
1926            $(
1927                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1928                impl $Trait<Self> for $ty {
1929                    type Result = $r;
1930                    #[inline]
1931                    #[target_feature(enable = "vector")]
1932                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
1933                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
1934                        (transmute(x), y)
1935                    }
1936                }
1937            )*
1938        };
1939        ([cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1940            impl_vfae! { [cc $Trait $m] $imm
1941                $b vector_signed_char
1942                $b vector_unsigned_char
1943                $b vector_bool_char
1944
1945                $h vector_signed_short
1946                $h vector_unsigned_short
1947                $h vector_bool_short
1948
1949                $f vector_signed_int
1950                $f vector_unsigned_int
1951                $f vector_bool_int
1952            }
1953        };
1954        ([cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
1955            $(
1956                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1957                impl $Trait<Self> for $ty {
1958                    type Result = t_b!($ty);
1959                    #[inline]
1960                    #[target_feature(enable = "vector")]
1961                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
1962                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
1963                        (transmute(x), y)
1964                    }
1965                }
1966            )*
1967        };
1968        ([idx $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1969            impl_vfae! { [idx $Trait $m] $imm
1970                $b vector_signed_char vector_signed_char
1971                $b vector_unsigned_char vector_unsigned_char
1972                $b vector_bool_char vector_unsigned_char
1973
1974                $h vector_signed_short vector_signed_short
1975                $h vector_unsigned_short vector_unsigned_short
1976                $h vector_bool_short vector_unsigned_short
1977
1978                $f vector_signed_int vector_signed_int
1979                $f vector_unsigned_int vector_unsigned_int
1980                $f vector_bool_int vector_unsigned_int
1981            }
1982        };
1983        ([idx $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
1984            $(
1985                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1986                impl $Trait<Self> for $ty {
1987                    type Result = $r;
1988                    #[inline]
1989                    #[target_feature(enable = "vector")]
1990                    unsafe fn $m(self, b: Self) -> Self::Result {
1991                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
1992                    }
1993                }
1994            )*
1995        };
1996        ([$Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1997            impl_vfae! { [$Trait $m] $imm
1998                $b vector_signed_char
1999                $b vector_unsigned_char
2000                $b vector_bool_char
2001
2002                $h vector_signed_short
2003                $h vector_unsigned_short
2004                $h vector_bool_short
2005
2006                $f vector_signed_int
2007                $f vector_unsigned_int
2008                $f vector_bool_int
2009            }
2010        };
2011        ([$Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
2012            $(
2013                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2014                impl $Trait<Self> for $ty {
2015                    type Result = t_b!($ty);
2016                    #[inline]
2017                    #[target_feature(enable = "vector")]
2018                    unsafe fn $m(self, b: Self) -> Self::Result {
2019                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
2020                    }
2021                }
2022            )*
2023        };
2024    }
2025
2026    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2027    pub trait VectorFindAnyEq<Other> {
2028        type Result;
2029        unsafe fn vec_find_any_eq(self, other: Other) -> Self::Result;
2030    }
2031
2032    impl_vfae! { [VectorFindAnyEq vec_find_any_eq] Eq vfaeb vfaeh vfaef }
2033
2034    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2035    pub trait VectorFindAnyNe<Other> {
2036        type Result;
2037        unsafe fn vec_find_any_ne(self, other: Other) -> Self::Result;
2038    }
2039
2040    impl_vfae! { [VectorFindAnyNe vec_find_any_ne] Ne vfaeb vfaeh vfaef }
2041
2042    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2043    pub trait VectorFindAnyEqOrZeroIdx<Other> {
2044        type Result;
2045        unsafe fn vec_find_any_eq_or_0_idx(self, other: Other) -> Self::Result;
2046    }
2047
2048    impl_vfae! { [idx VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx] EqIdx
2049        vfaezb vector_signed_char vector_signed_char
2050        vfaezb vector_unsigned_char vector_unsigned_char
2051        vfaezb vector_bool_char vector_unsigned_char
2052
2053        vfaezh vector_signed_short vector_signed_short
2054        vfaezh vector_unsigned_short vector_unsigned_short
2055        vfaezh vector_bool_short vector_unsigned_short
2056
2057        vfaezf vector_signed_int vector_signed_int
2058        vfaezf vector_unsigned_int vector_unsigned_int
2059        vfaezf vector_bool_int vector_unsigned_int
2060    }
2061
2062    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2063    pub trait VectorFindAnyNeOrZeroIdx<Other> {
2064        type Result;
2065        unsafe fn vec_find_any_ne_or_0_idx(self, other: Other) -> Self::Result;
2066    }
2067
2068    impl_vfae! { [idx VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx] NeIdx
2069        vfaezb vector_signed_char vector_signed_char
2070        vfaezb vector_unsigned_char vector_unsigned_char
2071        vfaezb vector_bool_char vector_unsigned_char
2072
2073        vfaezh vector_signed_short vector_signed_short
2074        vfaezh vector_unsigned_short vector_unsigned_short
2075        vfaezh vector_bool_short vector_unsigned_short
2076
2077        vfaezf vector_signed_int vector_signed_int
2078        vfaezf vector_unsigned_int vector_unsigned_int
2079        vfaezf vector_bool_int vector_unsigned_int
2080    }
2081
2082    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2083    pub trait VectorFindAnyEqIdx<Other> {
2084        type Result;
2085        unsafe fn vec_find_any_eq_idx(self, other: Other) -> Self::Result;
2086    }
2087
2088    impl_vfae! { [idx VectorFindAnyEqIdx vec_find_any_eq_idx] EqIdx vfaeb vfaeh vfaef }
2089
2090    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2091    pub trait VectorFindAnyNeIdx<Other> {
2092        type Result;
2093        unsafe fn vec_find_any_ne_idx(self, other: Other) -> Self::Result;
2094    }
2095
2096    impl_vfae! { [idx VectorFindAnyNeIdx vec_find_any_ne_idx] NeIdx vfaeb vfaeh vfaef }
2097
2098    macro_rules! vfaes_wrapper {
2099        ($($name:ident $ty:ident)*) => {
2100            $(
2101                #[inline]
2102                #[target_feature(enable = "vector")]
2103                #[cfg_attr(test, assert_instr($name, IMM = 0))]
2104                unsafe fn $name<const IMM: i32>(
2105                    a: $ty,
2106                    b: $ty,
2107                ) -> PackedTuple<$ty, i32> {
2108                    super::$name(a, b, IMM)
2109                }
2110            )*
2111        }
2112     }
2113
2114    vfaes_wrapper! {
2115        vfaebs vector_signed_char
2116        vfaehs vector_signed_short
2117        vfaefs vector_signed_int
2118
2119        vfaezbs vector_signed_char
2120        vfaezhs vector_signed_short
2121        vfaezfs vector_signed_int
2122    }
2123
2124    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2125    pub trait VectorFindAnyEqCC<Other> {
2126        type Result;
2127        unsafe fn vec_find_any_eq_cc(self, other: Other) -> (Self::Result, i32);
2128    }
2129
2130    impl_vfae! { [cc VectorFindAnyEqCC vec_find_any_eq_cc] Eq vfaebs vfaehs vfaefs }
2131
2132    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2133    pub trait VectorFindAnyNeCC<Other> {
2134        type Result;
2135        unsafe fn vec_find_any_ne_cc(self, other: Other) -> (Self::Result, i32);
2136    }
2137
2138    impl_vfae! { [cc VectorFindAnyNeCC vec_find_any_ne_cc] Ne vfaebs vfaehs vfaefs }
2139
2140    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2141    pub trait VectorFindAnyEqIdxCC<Other> {
2142        type Result;
2143        unsafe fn vec_find_any_eq_idx_cc(self, other: Other) -> (Self::Result, i32);
2144    }
2145
2146    impl_vfae! { [idx_cc VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc] EqIdx vfaebs vfaehs vfaefs }
2147
2148    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2149    pub trait VectorFindAnyNeIdxCC<Other> {
2150        type Result;
2151        unsafe fn vec_find_any_ne_idx_cc(self, other: Other) -> (Self::Result, i32);
2152    }
2153
2154    impl_vfae! { [idx_cc VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc] NeIdx vfaebs vfaehs vfaefs }
2155
2156    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2157    pub trait VectorFindAnyEqOrZeroIdxCC<Other> {
2158        type Result;
2159        unsafe fn vec_find_any_eq_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2160    }
2161
2162    impl_vfae! { [idx_cc VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc] EqIdx vfaezbs vfaezhs vfaezfs }
2163
2164    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2165    pub trait VectorFindAnyNeOrZeroIdxCC<Other> {
2166        type Result;
2167        unsafe fn vec_find_any_ne_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2168    }
2169
2170    impl_vfae! { [idx_cc VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc] NeIdx vfaezbs vfaezhs vfaezfs }
2171
2172    #[inline]
2173    #[target_feature(enable = "vector")]
2174    #[cfg_attr(test, assert_instr(vl))]
2175    unsafe fn test_vector_load(offset: isize, ptr: *const i32) -> vector_signed_int {
2176        ptr.byte_offset(offset)
2177            .cast::<vector_signed_int>()
2178            .read_unaligned()
2179    }
2180
2181    #[inline]
2182    #[target_feature(enable = "vector")]
2183    #[cfg_attr(test, assert_instr(vst))]
2184    unsafe fn test_vector_store(vector: vector_signed_int, offset: isize, ptr: *mut i32) {
2185        ptr.byte_offset(offset)
2186            .cast::<vector_signed_int>()
2187            .write_unaligned(vector)
2188    }
2189
2190    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2191    pub trait VectorLoad: Sized {
2192        type ElementType;
2193
2194        #[inline]
2195        #[target_feature(enable = "vector")]
2196        unsafe fn vec_xl(offset: isize, ptr: *const Self::ElementType) -> Self {
2197            ptr.byte_offset(offset).cast::<Self>().read_unaligned()
2198        }
2199
2200        unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self;
2201
2202        unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(
2203            ptr: *const Self::ElementType,
2204        ) -> MaybeUninit<Self>;
2205    }
2206
2207    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2208    pub trait VectorStore: Sized {
2209        type ElementType;
2210
2211        #[inline]
2212        #[target_feature(enable = "vector")]
2213        unsafe fn vec_xst(self, offset: isize, ptr: *mut Self::ElementType) {
2214            ptr.byte_offset(offset).cast::<Self>().write_unaligned(self)
2215        }
2216
2217        unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32);
2218    }
2219
2220    macro_rules! impl_load_store {
2221        ($($ty:ident)*) => {
2222            $(
2223                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2224                impl VectorLoad for t_t_l!($ty) {
2225                    type ElementType = $ty;
2226
2227                    #[inline]
2228                    #[target_feature(enable = "vector")]
2229                    unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self {
2230                        transmute(vll( byte_count, ptr.cast(),))
2231                    }
2232
2233                    #[inline]
2234                    #[target_feature(enable = "vector")]
2235                    unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(ptr: *const Self::ElementType) -> MaybeUninit<Self> {
2236                        transmute(vlbb(ptr.cast(), const { validate_block_boundary(BLOCK_BOUNDARY) }))
2237                    }
2238
2239                }
2240
2241                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2242                impl VectorStore for t_t_l!($ty) {
2243                    type ElementType = $ty;
2244
2245                    #[inline]
2246                    #[target_feature(enable = "vector")]
2247                    unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32) {
2248                        vstl(transmute(self), byte_count, ptr.cast())
2249                    }
2250                }
2251            )*
2252        }
2253    }
2254
2255    impl_load_store! { i8 u8 i16 u16 i32 u32 i64 u64 f32 f64 }
2256
2257    #[inline]
2258    #[target_feature(enable = "vector")]
2259    #[cfg_attr(test, assert_instr(vll))]
2260    unsafe fn test_vec_load_len(ptr: *const i32, byte_count: u32) -> vector_signed_int {
2261        vector_signed_int::vec_load_len(ptr, byte_count)
2262    }
2263
2264    #[inline]
2265    #[target_feature(enable = "vector")]
2266    #[cfg_attr(test, assert_instr("vlbb"))]
2267    unsafe fn test_vec_load_bndry(ptr: *const i32) -> MaybeUninit<vector_signed_int> {
2268        vector_signed_int::vec_load_bndry::<512>(ptr)
2269    }
2270
2271    #[inline]
2272    #[target_feature(enable = "vector")]
2273    #[cfg_attr(test, assert_instr(vst))]
2274    unsafe fn test_vec_store_len(vector: vector_signed_int, ptr: *mut i32, byte_count: u32) {
2275        vector.vec_store_len(ptr, byte_count)
2276    }
2277
2278    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2279    pub trait VectorLoadPair: Sized {
2280        type ElementType;
2281
2282        unsafe fn vec_load_pair(a: Self::ElementType, b: Self::ElementType) -> Self;
2283    }
2284
2285    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2286    impl VectorLoadPair for vector_signed_long_long {
2287        type ElementType = i64;
2288
2289        #[inline]
2290        #[target_feature(enable = "vector")]
2291        unsafe fn vec_load_pair(a: i64, b: i64) -> Self {
2292            vector_signed_long_long([a, b])
2293        }
2294    }
2295
2296    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2297    impl VectorLoadPair for vector_unsigned_long_long {
2298        type ElementType = u64;
2299
2300        #[inline]
2301        #[target_feature(enable = "vector")]
2302        unsafe fn vec_load_pair(a: u64, b: u64) -> Self {
2303            vector_unsigned_long_long([a, b])
2304        }
2305    }
2306
2307    #[inline]
2308    #[target_feature(enable = "vector")]
2309    unsafe fn pack<T, const N: usize>(a: T, b: T) -> T {
2310        simd_shuffle(a, b, const { ShuffleMask::<N>::pack() })
2311    }
2312
2313    #[inline]
2314    #[target_feature(enable = "vector")]
2315    #[cfg_attr(test, assert_instr(vpkh))]
2316    unsafe fn vpkh(a: i16x8, b: i16x8) -> i8x16 {
2317        let a: i8x16 = transmute(a);
2318        let b: i8x16 = transmute(b);
2319        simd_shuffle(a, b, const { ShuffleMask::<16>::pack() })
2320    }
2321    #[inline]
2322    #[target_feature(enable = "vector")]
2323    #[cfg_attr(test, assert_instr(vpkf))]
2324    unsafe fn vpkf(a: i32x4, b: i32x4) -> i16x8 {
2325        let a: i16x8 = transmute(a);
2326        let b: i16x8 = transmute(b);
2327        simd_shuffle(a, b, const { ShuffleMask::<8>::pack() })
2328    }
2329    #[inline]
2330    #[target_feature(enable = "vector")]
2331    #[cfg_attr(test, assert_instr(vpkg))]
2332    unsafe fn vpkg(a: i64x2, b: i64x2) -> i32x4 {
2333        let a: i32x4 = transmute(a);
2334        let b: i32x4 = transmute(b);
2335        simd_shuffle(a, b, const { ShuffleMask::<4>::pack() })
2336    }
2337
2338    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2339    pub trait VectorPack<Other> {
2340        type Result;
2341        unsafe fn vec_pack(self, b: Other) -> Self::Result;
2342    }
2343
2344    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2345    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2346    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_bool_short, vector_bool_short) -> vector_bool_char }
2347    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2348    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2349    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_bool_int, vector_bool_int) -> vector_bool_short }
2350    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2351    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2352    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_bool_long_long, vector_bool_long_long) -> vector_bool_int }
2353
2354    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2355    pub trait VectorPacks<Other> {
2356        type Result;
2357        unsafe fn vec_packs(self, b: Other) -> Self::Result;
2358    }
2359
2360    impl_vec_trait! { [VectorPacks vec_packs] vpksh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2361    impl_vec_trait! { [VectorPacks vec_packs] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2362    impl_vec_trait! { [VectorPacks vec_packs] vpksf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2363    impl_vec_trait! { [VectorPacks vec_packs] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2364    impl_vec_trait! { [VectorPacks vec_packs] vpksg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2365    impl_vec_trait! { [VectorPacks vec_packs] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2366
2367    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2368    pub trait VectorPacksu<Other> {
2369        type Result;
2370        unsafe fn vec_packsu(self, b: Other) -> Self::Result;
2371    }
2372
2373    unsafe fn simd_smax<T: Copy>(a: T, b: T) -> T {
2374        simd_select::<T, T>(simd_gt::<T, T>(a, b), a, b)
2375    }
2376
2377    #[inline]
2378    #[target_feature(enable = "vector")]
2379    #[cfg_attr(test, assert_instr(vpklsh))]
2380    unsafe fn vpacksuh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char {
2381        vpklsh(
2382            simd_smax(a, vector_signed_short([0; 8])),
2383            simd_smax(b, vector_signed_short([0; 8])),
2384        )
2385    }
2386    #[inline]
2387    #[target_feature(enable = "vector")]
2388    #[cfg_attr(test, assert_instr(vpklsf))]
2389    unsafe fn vpacksuf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short {
2390        vpklsf(
2391            simd_smax(a, vector_signed_int([0; 4])),
2392            simd_smax(b, vector_signed_int([0; 4])),
2393        )
2394    }
2395    #[inline]
2396    #[target_feature(enable = "vector")]
2397    #[cfg_attr(test, assert_instr(vpklsg))]
2398    unsafe fn vpacksug(
2399        a: vector_signed_long_long,
2400        b: vector_signed_long_long,
2401    ) -> vector_unsigned_int {
2402        vpklsg(
2403            simd_smax(a, vector_signed_long_long([0; 2])),
2404            simd_smax(b, vector_signed_long_long([0; 2])),
2405        )
2406    }
2407
2408    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuh (vector_signed_short, vector_signed_short) -> vector_unsigned_char }
2409    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2410    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuf (vector_signed_int, vector_signed_int) -> vector_unsigned_short }
2411    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2412    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksug (vector_signed_long_long, vector_signed_long_long) -> vector_unsigned_int }
2413    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2414
2415    macro_rules! impl_vector_packs_cc {
2416        ($($intr:ident $ty:ident $outty:ident)*) => {
2417            $(
2418                #[inline]
2419                #[target_feature(enable = "vector")]
2420                #[cfg_attr(test, assert_instr($intr))]
2421                unsafe fn $intr(
2422                    a: $ty,
2423                    b: $ty,
2424                ) -> ($outty, i32) {
2425                    let PackedTuple { x, y } = super::$intr(a, b);
2426                    (x, y)
2427                }
2428
2429                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2430                impl VectorPacksCC for $ty {
2431                    type Result = $outty;
2432
2433                    #[inline]
2434                    #[target_feature(enable = "vector")]
2435                    unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32) {
2436                        $intr(self, b)
2437                    }
2438                }
2439            )*
2440        }
2441    }
2442
2443    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2444    pub trait VectorPacksCC {
2445        type Result;
2446        unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32);
2447    }
2448
2449    impl_vector_packs_cc! {
2450        vpkshs vector_signed_short vector_signed_char
2451        vpklshs vector_unsigned_short vector_unsigned_char
2452        vpksfs vector_signed_int vector_signed_short
2453        vpklsfs vector_unsigned_int vector_unsigned_short
2454        vpksgs vector_signed_long_long vector_signed_int
2455        vpklsgs vector_unsigned_long_long vector_unsigned_int
2456    }
2457
2458    macro_rules! impl_vector_packsu_cc {
2459        ($($intr:ident $ty:ident $outty:ident)*) => {
2460            $(
2461                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2462                impl VectorPacksuCC for $ty {
2463                    type Result = $outty;
2464
2465                    #[inline]
2466                    #[target_feature(enable = "vector")]
2467                    unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32) {
2468                        $intr(self, b)
2469                    }
2470                }
2471            )*
2472        }
2473    }
2474
2475    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2476    pub trait VectorPacksuCC {
2477        type Result;
2478        unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32);
2479    }
2480
2481    impl_vector_packsu_cc! {
2482        vpklshs vector_unsigned_short vector_unsigned_char
2483        vpklsfs vector_unsigned_int vector_unsigned_short
2484        vpklsgs vector_unsigned_long_long vector_unsigned_int
2485    }
2486
2487    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2488    pub trait VectorMadd {
2489        unsafe fn vec_madd(self, b: Self, c: Self) -> Self;
2490        unsafe fn vec_msub(self, b: Self, c: Self) -> Self;
2491    }
2492
2493    test_impl! { vfmasb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fma, "vector-enhancements-1" vfmasb] }
2494    test_impl! { vfmadb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fma, vfmadb] }
2495
2496    #[inline]
2497    unsafe fn simd_fms<T>(a: T, b: T, c: T) -> T {
2498        simd_fma(a, b, simd_neg(c))
2499    }
2500
2501    test_impl! { vfmssb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fms, "vector-enhancements-1" vfmssb] }
2502    test_impl! { vfmsdb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fms, vfmsdb] }
2503
2504    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2505    impl VectorMadd for vector_float {
2506        #[inline]
2507        #[target_feature(enable = "vector")]
2508        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2509            vfmasb(self, b, c)
2510        }
2511
2512        #[inline]
2513        #[target_feature(enable = "vector")]
2514        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2515            vfmssb(self, b, c)
2516        }
2517    }
2518
2519    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2520    impl VectorMadd for vector_double {
2521        #[inline]
2522        #[target_feature(enable = "vector")]
2523        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2524            vfmadb(self, b, c)
2525        }
2526
2527        #[inline]
2528        #[target_feature(enable = "vector")]
2529        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2530            vfmsdb(self, b, c)
2531        }
2532    }
2533
2534    macro_rules! impl_vec_unpack {
2535        ($mask:ident $instr:ident $src:ident $shuffled:ident $dst:ident $width:literal) => {
2536            #[inline]
2537            #[target_feature(enable = "vector")]
2538            #[cfg_attr(test, assert_instr($instr))]
2539            unsafe fn $instr(a: $src) -> $dst {
2540                simd_as(simd_shuffle::<_, _, $shuffled>(
2541                    a,
2542                    a,
2543                    const { ShuffleMask::<$width>::$mask() },
2544                ))
2545            }
2546        };
2547    }
2548
2549    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2550    pub trait VectorUnpackh {
2551        type Result;
2552        unsafe fn vec_unpackh(self) -> Self::Result;
2553    }
2554
2555    impl_vec_unpack!(unpack_high vuphb vector_signed_char i8x8 vector_signed_short 8);
2556    impl_vec_unpack!(unpack_high vuphh vector_signed_short i16x4 vector_signed_int 4);
2557    impl_vec_unpack!(unpack_high vuphf vector_signed_int i32x2 vector_signed_long_long 2);
2558
2559    impl_vec_unpack!(unpack_high vuplhb vector_unsigned_char u8x8 vector_unsigned_short 8);
2560    impl_vec_unpack!(unpack_high vuplhh vector_unsigned_short u16x4 vector_unsigned_int 4);
2561    impl_vec_unpack!(unpack_high vuplhf vector_unsigned_int u32x2 vector_unsigned_long_long 2);
2562
2563    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphb (vector_signed_char) -> vector_signed_short}
2564    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphh (vector_signed_short) -> vector_signed_int}
2565    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphf (vector_signed_int) -> vector_signed_long_long}
2566
2567    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhb (vector_unsigned_char) -> vector_unsigned_short}
2568    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhh (vector_unsigned_short) -> vector_unsigned_int}
2569    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhf (vector_unsigned_int) -> vector_unsigned_long_long}
2570
2571    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhb (vector_bool_char) -> vector_bool_short}
2572    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhh (vector_bool_short) -> vector_bool_int}
2573    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhf (vector_bool_int) -> vector_bool_long_long}
2574
2575    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2576    pub trait VectorUnpackl {
2577        type Result;
2578        unsafe fn vec_unpackl(self) -> Self::Result;
2579    }
2580
2581    // FIXME(llvm): a shuffle + simd_as does not currently optimize into a single instruction like
2582    // unpachk above. Tracked in https://github.com/llvm/llvm-project/issues/129576.
2583
2584    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplbw (vector_signed_char) -> vector_signed_short}
2585    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplhw (vector_signed_short) -> vector_signed_int}
2586    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplfw (vector_signed_int) -> vector_signed_long_long}
2587
2588    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllb (vector_unsigned_char) -> vector_unsigned_short}
2589    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllh (vector_unsigned_short) -> vector_unsigned_int}
2590    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllf (vector_unsigned_int) -> vector_unsigned_long_long}
2591
2592    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllb (vector_bool_char) -> vector_bool_short}
2593    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllh (vector_bool_short) -> vector_bool_int}
2594    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllf (vector_bool_int) -> vector_bool_long_long}
2595
2596    test_impl! { vec_vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vavgb, vavgb ] }
2597    test_impl! { vec_vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vavgh, vavgh ] }
2598    test_impl! { vec_vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vavgf, vavgf ] }
2599    test_impl! { vec_vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [ vavgg, vavgg ] }
2600
2601    test_impl! { vec_vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vavglb, vavglb ] }
2602    test_impl! { vec_vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vavglh, vavglh ] }
2603    test_impl! { vec_vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vavglf, vavglf ] }
2604    test_impl! { vec_vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [ vavglg, vavglg ] }
2605
2606    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2607    pub trait VectorAvg<Other> {
2608        type Result;
2609        unsafe fn vec_avg(self, b: Other) -> Self::Result;
2610    }
2611
2612    impl_vec_trait! { [VectorAvg vec_avg] 2 (vec_vavglb, vec_vavgb, vec_vavglh, vec_vavgh, vec_vavglf, vec_vavgf, vec_vavglg, vec_vavgg) }
2613
2614    macro_rules! impl_mul {
2615        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty) -> $r:ty) => {
2616            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2617            impl $Trait<$r> for $a {
2618                #[inline]
2619                #[target_feature(enable = "vector")]
2620                unsafe fn $m(self, b: $b) -> $r {
2621                    $fun(transmute(self), transmute(b))
2622                }
2623            }
2624        };
2625        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty, $c:ty) -> $r:ty) => {
2626            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2627            impl $Trait for $a {
2628                type Result = $r;
2629                #[inline]
2630                #[target_feature(enable = "vector")]
2631                unsafe fn $m(self, b: $b, c: $c) -> $r {
2632                    $fun(self, b, c)
2633                }
2634            }
2635        };
2636    }
2637
2638    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2639    pub trait VectorMule<Result> {
2640        unsafe fn vec_mule(self, b: Self) -> Result;
2641    }
2642
2643    // FIXME(llvm) sadly this does not yet work https://github.com/llvm/llvm-project/issues/129705
2644    //    #[target_feature(enable = "vector")]
2645    //    #[cfg_attr(test, assert_instr(vmleh))]
2646    //    unsafe fn vec_vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int {
2647    //        let even_a: vector_unsigned_int = simd_as(simd_shuffle::<_, _, u16x4>(
2648    //            a,
2649    //            a,
2650    //            const { ShuffleMask([0, 2, 4, 6]) },
2651    //        ));
2652    //
2653    //        let even_b: vector_unsigned_int = simd_as(simd_shuffle::<_, _, u16x4>(
2654    //            b,
2655    //            b,
2656    //            const { ShuffleMask([0, 2, 4, 6]) },
2657    //        ));
2658    //
2659    //        simd_mul(even_a, even_b)
2660    //    }
2661
2662    test_impl! { vec_vmeb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmeb, vmeb ] }
2663    test_impl! { vec_vmeh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmeh, vmeh ] }
2664    test_impl! { vec_vmef(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmef, vmef ] }
2665
2666    test_impl! { vec_vmleb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmleb, vmleb ] }
2667    test_impl! { vec_vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmleh, vmleh ] }
2668    test_impl! { vec_vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlef, vmlef ] }
2669
2670    impl_mul!([VectorMule vec_mule] vec_vmeb (vector_signed_char, vector_signed_char) -> vector_signed_short );
2671    impl_mul!([VectorMule vec_mule] vec_vmeh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2672    impl_mul!([VectorMule vec_mule] vec_vmef (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2673
2674    impl_mul!([VectorMule vec_mule] vec_vmleb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2675    impl_mul!([VectorMule vec_mule] vec_vmleh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2676    impl_mul!([VectorMule vec_mule] vec_vmlef (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2677
2678    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2679    pub trait VectorMulo<Result> {
2680        unsafe fn vec_mulo(self, b: Self) -> Result;
2681    }
2682
2683    test_impl! { vec_vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmob, vmob ] }
2684    test_impl! { vec_vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmoh, vmoh ] }
2685    test_impl! { vec_vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmof, vmof ] }
2686
2687    test_impl! { vec_vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmlob, vmlob ] }
2688    test_impl! { vec_vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmloh, vmloh ] }
2689    test_impl! { vec_vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlof, vmlof ] }
2690
2691    impl_mul!([VectorMulo vec_mulo] vec_vmob (vector_signed_char, vector_signed_char) -> vector_signed_short );
2692    impl_mul!([VectorMulo vec_mulo] vec_vmoh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2693    impl_mul!([VectorMulo vec_mulo] vec_vmof (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2694
2695    impl_mul!([VectorMulo vec_mulo] vec_vmlob (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2696    impl_mul!([VectorMulo vec_mulo] vec_vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2697    impl_mul!([VectorMulo vec_mulo] vec_vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2698
2699    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2700    pub trait VectorMulh<Result> {
2701        unsafe fn vec_mulh(self, b: Self) -> Result;
2702    }
2703
2704    test_impl! { vec_vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vmhb, vmhb ] }
2705    test_impl! { vec_vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vmhh, vmhh ] }
2706    test_impl! { vec_vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vmhf, vmhf ] }
2707
2708    test_impl! { vec_vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vmlhb, vmlhb ] }
2709    test_impl! { vec_vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vmlhh, vmlhh ] }
2710    test_impl! { vec_vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vmlhf, vmlhf ] }
2711
2712    impl_mul!([VectorMulh vec_mulh] vec_vmhb (vector_signed_char, vector_signed_char) -> vector_signed_char);
2713    impl_mul!([VectorMulh vec_mulh] vec_vmhh (vector_signed_short, vector_signed_short) -> vector_signed_short);
2714    impl_mul!([VectorMulh vec_mulh] vec_vmhf (vector_signed_int, vector_signed_int) -> vector_signed_int);
2715
2716    impl_mul!([VectorMulh vec_mulh] vec_vmlhb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char);
2717    impl_mul!([VectorMulh vec_mulh] vec_vmlhh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2718    impl_mul!([VectorMulh vec_mulh] vec_vmlhf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int);
2719
2720    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2721    pub trait VectorMeadd {
2722        type Result;
2723        unsafe fn vec_meadd(self, b: Self, c: Self::Result) -> Self::Result;
2724    }
2725
2726    test_impl! { vec_vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaeb, vmaeb ] }
2727    test_impl! { vec_vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaeh, vmaeh ] }
2728    test_impl! { vec_vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaef, vmaef ] }
2729
2730    test_impl! { vec_vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmaleb, vmaleb ] }
2731    test_impl! { vec_vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaleh, vmaleh ] }
2732    test_impl! { vec_vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalef, vmalef ] }
2733
2734    impl_mul!([VectorMeadd vec_meadd] vec_vmaeb (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2735    impl_mul!([VectorMeadd vec_meadd] vec_vmaeh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2736    impl_mul!([VectorMeadd vec_meadd] vec_vmaef (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2737
2738    impl_mul!([VectorMeadd vec_meadd] vec_vmaleb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2739    impl_mul!([VectorMeadd vec_meadd] vec_vmaleh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2740    impl_mul!([VectorMeadd vec_meadd] vec_vmalef (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2741
2742    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2743    pub trait VectorMoadd {
2744        type Result;
2745        unsafe fn vec_moadd(self, b: Self, c: Self::Result) -> Self::Result;
2746    }
2747
2748    test_impl! { vec_vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaob, vmaob ] }
2749    test_impl! { vec_vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaoh, vmaoh ] }
2750    test_impl! { vec_vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaof, vmaof ] }
2751
2752    test_impl! { vec_vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmalob, vmalob ] }
2753    test_impl! { vec_vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaloh, vmaloh ] }
2754    test_impl! { vec_vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalof, vmalof ] }
2755
2756    impl_mul!([VectorMoadd vec_moadd] vec_vmaob (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2757    impl_mul!([VectorMoadd vec_moadd] vec_vmaoh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2758    impl_mul!([VectorMoadd vec_moadd] vec_vmaof (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2759
2760    impl_mul!([VectorMoadd vec_moadd] vec_vmalob (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2761    impl_mul!([VectorMoadd vec_moadd] vec_vmaloh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2762    impl_mul!([VectorMoadd vec_moadd] vec_vmalof (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2763
2764    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2765    pub trait VectorMhadd {
2766        type Result;
2767        unsafe fn vec_mhadd(self, b: Self, c: Self::Result) -> Self::Result;
2768    }
2769
2770    test_impl! { vec_vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [ vmahb, vmahb ] }
2771    test_impl! { vec_vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[ vmahh, vmahh ] }
2772    test_impl! { vec_vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [ vmahf, vmahf ] }
2773
2774    test_impl! { vec_vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [ vmalhb, vmalhb ] }
2775    test_impl! { vec_vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[ vmalhh, vmalhh ] }
2776    test_impl! { vec_vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [ vmalhf, vmalhf ] }
2777
2778    impl_mul!([VectorMhadd vec_mhadd] vec_vmahb (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2779    impl_mul!([VectorMhadd vec_mhadd] vec_vmahh (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2780    impl_mul!([VectorMhadd vec_mhadd] vec_vmahf (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2781
2782    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2783    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2784    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2785
2786    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2787    pub trait VectorMladd {
2788        type Result;
2789        unsafe fn vec_mladd(self, b: Self, c: Self::Result) -> Self::Result;
2790    }
2791
2792    #[inline]
2793    #[target_feature(enable = "vector")]
2794    unsafe fn simd_mladd<T>(a: T, b: T, c: T) -> T {
2795        simd_add(simd_mul(a, b), c)
2796    }
2797
2798    test_impl! { vec_vmal_ib(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [simd_mladd, vmalb ] }
2799    test_impl! { vec_vmal_ih(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[simd_mladd, vmalh ] }
2800    test_impl! { vec_vmal_if(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [simd_mladd, vmalf ] }
2801
2802    test_impl! { vec_vmal_ub(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [simd_mladd, vmalb ] }
2803    test_impl! { vec_vmal_uh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[simd_mladd, vmalh ] }
2804    test_impl! { vec_vmal_uf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [simd_mladd, vmalf ] }
2805
2806    impl_mul!([VectorMladd vec_mladd] vec_vmal_ib (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2807    impl_mul!([VectorMladd vec_mladd] vec_vmal_ih (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2808    impl_mul!([VectorMladd vec_mladd] vec_vmal_if (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2809
2810    impl_mul!([VectorMladd vec_mladd] vec_vmal_ub (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2811    impl_mul!([VectorMladd vec_mladd] vec_vmal_uh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2812    impl_mul!([VectorMladd vec_mladd] vec_vmal_uf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2813
2814    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2815    pub trait VectorGfmsum<Result> {
2816        unsafe fn vec_gfmsum(self, b: Self) -> Result;
2817    }
2818
2819    test_impl! { vec_vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vgfmb, vgfmb ] }
2820    test_impl! { vec_vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vgfmh, vgfmh] }
2821    test_impl! { vec_vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vgfmf, vgfmf ] }
2822
2823    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2824    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2825    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2826
2827    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2828    pub trait VectorGfmsumAccum {
2829        type Result;
2830        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result;
2831    }
2832
2833    test_impl! { vec_vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vgfmab, vgfmab ] }
2834    test_impl! { vec_vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vgfmah, vgfmah] }
2835    test_impl! { vec_vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vgfmaf, vgfmaf ] }
2836
2837    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2838    impl VectorGfmsumAccum for vector_unsigned_char {
2839        type Result = vector_unsigned_short;
2840        #[inline]
2841        #[target_feature(enable = "vector")]
2842        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2843            vec_vgfmab(self, b, c)
2844        }
2845    }
2846    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2847    impl VectorGfmsumAccum for vector_unsigned_short {
2848        type Result = vector_unsigned_int;
2849        #[inline]
2850        #[target_feature(enable = "vector")]
2851        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2852            vec_vgfmah(self, b, c)
2853        }
2854    }
2855    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2856    impl VectorGfmsumAccum for vector_unsigned_int {
2857        type Result = vector_unsigned_long_long;
2858        #[inline]
2859        #[target_feature(enable = "vector")]
2860        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2861            vec_vgfmaf(self, b, c)
2862        }
2863    }
2864
2865    #[inline]
2866    #[target_feature(enable = "vector")]
2867    #[cfg_attr(test, assert_instr(vgef, D = 3))]
2868    unsafe fn vgef<const D: u32>(
2869        a: vector_unsigned_int,
2870        b: vector_unsigned_int,
2871        c: *const u32,
2872    ) -> vector_unsigned_int {
2873        static_assert_uimm_bits!(D, 2);
2874        let offset: u32 = simd_extract(b, D);
2875        let ptr = c.byte_add(offset as usize);
2876        let value = ptr.read();
2877        simd_insert(a, D, value)
2878    }
2879
2880    #[inline]
2881    #[target_feature(enable = "vector")]
2882    #[cfg_attr(test, assert_instr(vgeg, D = 1))]
2883    unsafe fn vgeg<const D: u32>(
2884        a: vector_unsigned_long_long,
2885        b: vector_unsigned_long_long,
2886        c: *const u64,
2887    ) -> vector_unsigned_long_long {
2888        static_assert_uimm_bits!(D, 1);
2889        let offset: u64 = simd_extract(b, D);
2890        let ptr = c.byte_add(offset as usize);
2891        let value = ptr.read();
2892        simd_insert(a, D, value)
2893    }
2894
2895    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2896    pub trait VectorGatherElement {
2897        type Element;
2898        type Offset;
2899        unsafe fn vec_gather_element<const D: u32>(
2900            self,
2901            b: Self::Offset,
2902            c: *const Self::Element,
2903        ) -> Self;
2904    }
2905
2906    macro_rules! impl_vec_gather_element {
2907        ($($instr:ident $ty:ident)*) => {
2908            $(
2909                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2910                impl VectorGatherElement for $ty {
2911                    type Element = l_t_t!($ty);
2912                    type Offset = t_u!($ty);
2913
2914                    #[inline]
2915                    #[target_feature(enable = "vector")]
2916                    unsafe fn vec_gather_element<const D: u32>(self, b: Self::Offset, c: *const Self::Element) -> Self {
2917                        transmute($instr::<D>(transmute(self), b, c.cast()))
2918                    }
2919                }
2920            )*
2921        }
2922    }
2923
2924    impl_vec_gather_element! {
2925        vgef vector_signed_int
2926        vgef vector_bool_int
2927        vgef vector_unsigned_int
2928
2929        vgeg vector_signed_long_long
2930        vgeg vector_bool_long_long
2931        vgeg vector_unsigned_long_long
2932
2933        vgef vector_float
2934        vgeg vector_double
2935    }
2936
2937    #[inline]
2938    #[target_feature(enable = "vector")]
2939    #[cfg_attr(test, assert_instr(vscef, D = 3))]
2940    unsafe fn vscef<const D: u32>(a: vector_unsigned_int, b: vector_unsigned_int, c: *mut u32) {
2941        static_assert_uimm_bits!(D, 2);
2942        let value = simd_extract(a, D);
2943        let offset: u32 = simd_extract(b, D);
2944        let ptr = c.byte_add(offset as usize);
2945        ptr.write(value);
2946    }
2947
2948    #[inline]
2949    #[target_feature(enable = "vector")]
2950    #[cfg_attr(test, assert_instr(vsceg, D = 1))]
2951    unsafe fn vsceg<const D: u32>(
2952        a: vector_unsigned_long_long,
2953        b: vector_unsigned_long_long,
2954        c: *mut u64,
2955    ) {
2956        static_assert_uimm_bits!(D, 1);
2957        let value = simd_extract(a, D);
2958        let offset: u64 = simd_extract(b, D);
2959        let ptr = c.byte_add(offset as usize);
2960        ptr.write(value);
2961    }
2962
2963    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2964    pub trait VectorScatterElement {
2965        type Element;
2966        type Offset;
2967        unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element);
2968    }
2969
2970    macro_rules! impl_vec_scatter_element {
2971        ($($instr:ident $ty:ident)*) => {
2972            $(
2973                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2974                impl VectorScatterElement for $ty {
2975                    type Element = l_t_t!($ty);
2976                    type Offset = t_u!($ty);
2977
2978                    #[inline]
2979                    #[target_feature(enable = "vector")]
2980                    unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element) {
2981                        $instr::<D>(transmute(self), b, c.cast())
2982                    }
2983                }
2984            )*
2985        }
2986    }
2987
2988    impl_vec_scatter_element! {
2989        vscef vector_signed_int
2990        vscef vector_bool_int
2991        vscef vector_unsigned_int
2992
2993        vsceg vector_signed_long_long
2994        vsceg vector_bool_long_long
2995        vsceg vector_unsigned_long_long
2996
2997        vscef vector_float
2998        vsceg vector_double
2999    }
3000
3001    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3002    pub trait VectorSel<Mask>: Sized {
3003        unsafe fn vec_sel(self, b: Self, c: Mask) -> Self;
3004    }
3005
3006    macro_rules! impl_vec_sel {
3007        ($($ty:ident)*) => {
3008            $(
3009                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3010                impl VectorSel<t_u!($ty)> for $ty {
3011                    #[inline]
3012                    #[target_feature(enable = "vector")]
3013                    unsafe fn vec_sel(self, b: Self, c: t_u!($ty)) -> Self {
3014                        let b = simd_and(b, transmute(c));
3015                        let a = simd_and(self, simd_xor(transmute(c), transmute(vector_signed_char([!0; 16]))));
3016                        simd_or(a, b)
3017                    }
3018                }
3019
3020                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3021                impl VectorSel<t_b!($ty)> for $ty {
3022                    #[inline]
3023                    #[target_feature(enable = "vector")]
3024                    unsafe fn vec_sel(self, b: Self, c: t_b!($ty)) -> Self {
3025                        // defer to the implementation with an unsigned mask
3026                        self.vec_sel(b, transmute::<_, t_u!($ty)>(c))
3027                    }
3028                }
3029            )*
3030        }
3031    }
3032
3033    impl_vec_sel! {
3034        vector_signed_char
3035        vector_signed_short
3036        vector_signed_int
3037        vector_signed_long_long
3038
3039        vector_unsigned_char
3040        vector_unsigned_short
3041        vector_unsigned_int
3042        vector_unsigned_long_long
3043
3044        vector_bool_char
3045        vector_bool_short
3046        vector_bool_int
3047        vector_bool_long_long
3048
3049        vector_float
3050        vector_double
3051    }
3052
3053    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3054    pub trait VectorFpTestDataClass {
3055        type Result;
3056        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32);
3057    }
3058
3059    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3060    impl VectorFpTestDataClass for vector_float {
3061        type Result = vector_bool_int;
3062
3063        #[inline]
3064        #[target_feature(enable = "vector")]
3065        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3066            let PackedTuple { x, y } = vftcisb(self, CLASS);
3067            (x, y)
3068        }
3069    }
3070
3071    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3072    impl VectorFpTestDataClass for vector_double {
3073        type Result = vector_bool_long_long;
3074
3075        #[inline]
3076        #[target_feature(enable = "vector")]
3077        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3078            let PackedTuple { x, y } = vftcidb(self, CLASS);
3079            (x, y)
3080        }
3081    }
3082
3083    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3084    pub trait VectorCompare {
3085        unsafe fn vec_all_lt(self, other: Self) -> i32;
3086        unsafe fn vec_all_le(self, other: Self) -> i32;
3087        unsafe fn vec_all_gt(self, other: Self) -> i32;
3088        unsafe fn vec_all_ge(self, other: Self) -> i32;
3089    }
3090
3091    // NOTE: this implementation is currently non-optimal, but it does work for floats even with
3092    // only `vector` enabled.
3093    //
3094    // - https://github.com/llvm/llvm-project/issues/129434
3095    // - https://github.com/llvm/llvm-project/issues/130424
3096    macro_rules! impl_vec_compare {
3097        ($($ty:ident)*) => {
3098            $(
3099                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3100                impl VectorCompare for $ty {
3101                    #[inline]
3102                    #[target_feature(enable = "vector")]
3103                    unsafe fn vec_all_lt(self, other: Self) -> i32 {
3104                        simd_reduce_all(simd_lt::<_, t_b!($ty)>(self, other)) as i32
3105                    }
3106                    #[inline]
3107                    #[target_feature(enable = "vector")]
3108                    unsafe fn vec_all_le(self, other: Self) -> i32 {
3109                        simd_reduce_all(simd_le::<_, t_b!($ty)>(self, other)) as i32
3110                    }
3111                    #[inline]
3112                    #[target_feature(enable = "vector")]
3113                    unsafe fn vec_all_gt(self, other: Self) -> i32 {
3114                        simd_reduce_all(simd_gt::<_, t_b!($ty)>(self, other)) as i32
3115                    }
3116                    #[inline]
3117                    #[target_feature(enable = "vector")]
3118                    unsafe fn vec_all_ge(self, other: Self) -> i32 {
3119                        simd_reduce_all(simd_ge::<_, t_b!($ty)>(self, other)) as i32
3120                    }
3121                }
3122            )*
3123        }
3124    }
3125
3126    impl_vec_compare! {
3127        vector_signed_char
3128        vector_unsigned_char
3129
3130        vector_signed_short
3131        vector_unsigned_short
3132
3133        vector_signed_int
3134        vector_unsigned_int
3135        vector_float
3136
3137        vector_signed_long_long
3138        vector_unsigned_long_long
3139        vector_double
3140    }
3141
3142    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3143    pub trait VectorTestMask {
3144        type Mask;
3145        unsafe fn vec_test_mask(self, other: Self::Mask) -> i32;
3146    }
3147
3148    macro_rules! impl_vec_test_mask {
3149        ($($instr:ident $ty:ident)*) => {
3150            $(
3151                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3152                impl VectorTestMask for $ty {
3153                    type Mask = t_u!($ty);
3154
3155                    #[inline]
3156                    #[target_feature(enable = "vector")]
3157                    unsafe fn vec_test_mask(self, other: Self::Mask) -> i32 {
3158                        vtm(transmute(self), transmute(other))
3159                    }
3160                }
3161            )*
3162        }
3163    }
3164
3165    impl_vec_test_mask! {
3166        vector_signed_char
3167        vector_signed_short
3168        vector_signed_int
3169        vector_signed_long_long
3170
3171        vector_unsigned_char
3172        vector_unsigned_short
3173        vector_unsigned_int
3174        vector_unsigned_long_long
3175
3176        vector_float
3177        vector_double
3178    }
3179
3180    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3181    pub trait VectorSearchString {
3182        unsafe fn vec_search_string_cc(
3183            self,
3184            b: Self,
3185            c: vector_unsigned_char,
3186        ) -> (vector_unsigned_char, i32);
3187
3188        unsafe fn vec_search_string_until_zero_cc(
3189            self,
3190            b: Self,
3191            c: vector_unsigned_char,
3192        ) -> (vector_unsigned_char, i32);
3193    }
3194
3195    macro_rules! impl_vec_search_string{
3196        ($($intr_s:ident $intr_sz:ident $ty:ident)*) => {
3197            $(
3198                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3199                impl VectorSearchString for $ty {
3200                    #[inline]
3201                    #[target_feature(enable = "vector")]
3202                    unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3203                        let PackedTuple { x,y } = $intr_s(transmute(self), transmute(b), c);
3204                        (x, y)
3205                    }
3206
3207                    #[inline]
3208                    #[target_feature(enable = "vector")]
3209                    unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3210                        let PackedTuple { x,y } = $intr_sz(transmute(self), transmute(b), c);
3211                        (x, y)
3212                    }
3213                }
3214
3215            )*
3216        }
3217    }
3218
3219    impl_vec_search_string! {
3220        vstrsb vstrszb vector_signed_char
3221        vstrsb vstrszb vector_bool_char
3222        vstrsb vstrszb vector_unsigned_char
3223
3224        vstrsh vstrszh vector_signed_short
3225        vstrsh vstrszh vector_bool_short
3226        vstrsh vstrszh vector_unsigned_short
3227
3228        vstrsf vstrszf vector_signed_int
3229        vstrsf vstrszf vector_bool_int
3230        vstrsf vstrszf vector_unsigned_int
3231    }
3232
3233    #[inline]
3234    #[target_feature(enable = "vector")]
3235    #[cfg_attr(test, assert_instr(vcdgb))]
3236    pub unsafe fn vcdgb(a: vector_signed_long_long) -> vector_double {
3237        simd_as(a)
3238    }
3239
3240    #[inline]
3241    #[target_feature(enable = "vector")]
3242    #[cfg_attr(test, assert_instr(vcdlgb))]
3243    pub unsafe fn vcdlgb(a: vector_unsigned_long_long) -> vector_double {
3244        simd_as(a)
3245    }
3246
3247    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3248    pub trait VectorDouble {
3249        unsafe fn vec_double(self) -> vector_double;
3250    }
3251
3252    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3253    impl VectorDouble for vector_signed_long_long {
3254        #[inline]
3255        #[target_feature(enable = "vector")]
3256        unsafe fn vec_double(self) -> vector_double {
3257            vcdgb(self)
3258        }
3259    }
3260
3261    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3262    impl VectorDouble for vector_unsigned_long_long {
3263        #[inline]
3264        #[target_feature(enable = "vector")]
3265        unsafe fn vec_double(self) -> vector_double {
3266            vcdlgb(self)
3267        }
3268    }
3269
3270    #[inline]
3271    #[target_feature(enable = "vector")]
3272    #[cfg_attr(
3273        all(test, target_feature = "vector-enhancements-2"),
3274        assert_instr(vcefb)
3275    )]
3276    pub unsafe fn vcefb(a: vector_signed_int) -> vector_float {
3277        simd_as(a)
3278    }
3279
3280    #[inline]
3281    #[target_feature(enable = "vector")]
3282    #[cfg_attr(
3283        all(test, target_feature = "vector-enhancements-2"),
3284        assert_instr(vcelfb)
3285    )]
3286    pub unsafe fn vcelfb(a: vector_unsigned_int) -> vector_float {
3287        simd_as(a)
3288    }
3289
3290    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3291    pub trait VectorFloat {
3292        unsafe fn vec_float(self) -> vector_float;
3293    }
3294
3295    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3296    impl VectorFloat for vector_signed_int {
3297        #[inline]
3298        #[target_feature(enable = "vector")]
3299        unsafe fn vec_float(self) -> vector_float {
3300            vcefb(self)
3301        }
3302    }
3303
3304    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3305    impl VectorFloat for vector_unsigned_int {
3306        #[inline]
3307        #[target_feature(enable = "vector")]
3308        unsafe fn vec_float(self) -> vector_float {
3309            vcelfb(self)
3310        }
3311    }
3312
3313    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3314    pub trait VectorExtendSigned64 {
3315        unsafe fn vec_extend_s64(self) -> vector_signed_long_long;
3316    }
3317
3318    #[inline]
3319    #[target_feature(enable = "vector")]
3320    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3321    // #[cfg_attr(test, assert_instr(vsegb))]
3322    pub unsafe fn vsegb(a: vector_signed_char) -> vector_signed_long_long {
3323        simd_as(simd_shuffle::<_, _, i8x2>(
3324            a,
3325            a,
3326            const { u32x2::from_array([7, 15]) },
3327        ))
3328    }
3329
3330    #[inline]
3331    #[target_feature(enable = "vector")]
3332    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3333    // #[cfg_attr(test, assert_instr(vsegh))]
3334    pub unsafe fn vsegh(a: vector_signed_short) -> vector_signed_long_long {
3335        simd_as(simd_shuffle::<_, _, i16x2>(
3336            a,
3337            a,
3338            const { u32x2::from_array([3, 7]) },
3339        ))
3340    }
3341
3342    #[inline]
3343    #[target_feature(enable = "vector")]
3344    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3345    // #[cfg_attr(test, assert_instr(vsegf))]
3346    pub unsafe fn vsegf(a: vector_signed_int) -> vector_signed_long_long {
3347        simd_as(simd_shuffle::<_, _, i32x2>(
3348            a,
3349            a,
3350            const { u32x2::from_array([1, 3]) },
3351        ))
3352    }
3353
3354    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3355    impl VectorExtendSigned64 for vector_signed_char {
3356        #[inline]
3357        #[target_feature(enable = "vector")]
3358        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3359            vsegb(self)
3360        }
3361    }
3362    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3363    impl VectorExtendSigned64 for vector_signed_short {
3364        #[inline]
3365        #[target_feature(enable = "vector")]
3366        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3367            vsegh(self)
3368        }
3369    }
3370    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3371    impl VectorExtendSigned64 for vector_signed_int {
3372        #[inline]
3373        #[target_feature(enable = "vector")]
3374        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3375            vsegf(self)
3376        }
3377    }
3378
3379    // NOTE: VectorSigned and VectorUnsigned make strong safety assumptions around floats.
3380    // This is what C provides, but even IBM does not clearly document these constraints.
3381    //
3382    // https://doc.rust-lang.org/std/intrinsics/simd/fn.simd_cast.html
3383
3384    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3385    pub trait VectorSigned {
3386        type Result;
3387        unsafe fn vec_signed(self) -> Self::Result;
3388    }
3389
3390    test_impl! { vcgsb (a: vector_float) -> vector_signed_int [simd_cast, "vector-enhancements-2" vcgsb] }
3391    test_impl! { vcgdb (a: vector_double) -> vector_signed_long_long [simd_cast, vcgdb] }
3392
3393    impl_vec_trait! { [VectorSigned vec_signed] vcgsb (vector_float) -> vector_signed_int }
3394    impl_vec_trait! { [VectorSigned vec_signed] vcgdb (vector_double) -> vector_signed_long_long }
3395
3396    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3397    pub trait VectorUnsigned {
3398        type Result;
3399        unsafe fn vec_unsigned(self) -> Self::Result;
3400    }
3401
3402    test_impl! { vclgsb (a: vector_float) -> vector_unsigned_int [simd_cast, "vector-enhancements-2" vclgsb] }
3403    test_impl! { vclgdb (a: vector_double) -> vector_unsigned_long_long [simd_cast, vclgdb] }
3404
3405    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgsb (vector_float) -> vector_unsigned_int }
3406    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgdb (vector_double) -> vector_unsigned_long_long }
3407
3408    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3409    pub trait VectorCopyUntilZero {
3410        unsafe fn vec_cp_until_zero(self) -> Self;
3411    }
3412
3413    test_impl! { vec_vistrb (a: vector_unsigned_char) -> vector_unsigned_char [vistrb, vistrb] }
3414    test_impl! { vec_vistrh (a: vector_unsigned_short) -> vector_unsigned_short [vistrh, vistrh] }
3415    test_impl! { vec_vistrf (a: vector_unsigned_int) -> vector_unsigned_int [vistrf, vistrf] }
3416
3417    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_signed_char) }
3418    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_bool_char) }
3419    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_unsigned_char) }
3420
3421    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_signed_short) }
3422    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_bool_short) }
3423    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_unsigned_short) }
3424
3425    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_signed_int) }
3426    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_bool_int) }
3427    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_unsigned_int) }
3428
3429    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3430    pub trait VectorCopyUntilZeroCC: Sized {
3431        unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32);
3432    }
3433
3434    test_impl! { vec_vistrbs (a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32> [vistrbs, vistrbs] }
3435    test_impl! { vec_vistrhs (a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32> [vistrhs, vistrhs] }
3436    test_impl! { vec_vistrfs (a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32> [vistrfs, vistrfs] }
3437
3438    macro_rules! impl_vec_copy_until_zero_cc {
3439        ($($intr:ident $ty:ident)*) => {
3440            $(
3441                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3442                impl VectorCopyUntilZeroCC for $ty {
3443                    #[inline]
3444                    #[target_feature(enable = "vector")]
3445                    unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32) {
3446                        let PackedTuple { x,y } = $intr(transmute(self));
3447                        (transmute(x), y)
3448                    }
3449                }
3450
3451            )*
3452        }
3453    }
3454
3455    impl_vec_copy_until_zero_cc! {
3456        vec_vistrbs vector_signed_char
3457        vec_vistrbs vector_bool_char
3458        vec_vistrbs vector_unsigned_char
3459
3460        vec_vistrhs vector_signed_short
3461        vec_vistrhs vector_bool_short
3462        vec_vistrhs vector_unsigned_short
3463
3464        vec_vistrfs vector_signed_int
3465        vec_vistrfs vector_bool_int
3466        vec_vistrfs vector_unsigned_int
3467    }
3468
3469    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3470    pub trait VectorSrdb {
3471        unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self;
3472    }
3473
3474    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3475    pub trait VectorSld {
3476        unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self;
3477
3478        unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self;
3479
3480        unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self;
3481    }
3482
3483    // FIXME(llvm) https://github.com/llvm/llvm-project/issues/129955
3484    // ideally we could implement this in terms of llvm.fshl.i128
3485    // #[link_name = "llvm.fshl.i128"] fn fshl_i128(a: u128, b: u128, c: u128) -> u128;
3486    // transmute(fshl_i128(transmute(a), transmute(b), const { C * 8 } ))
3487
3488    macro_rules! impl_vec_sld {
3489        ($($ty:ident)*) => {
3490            $(
3491                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3492                impl VectorSld for $ty {
3493                    #[inline]
3494                    #[target_feature(enable = "vector")]
3495                    unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self {
3496                        static_assert_uimm_bits!(C, 4);
3497                        transmute(vsldb(transmute(self), transmute(b), C))
3498                    }
3499
3500                    #[inline]
3501                    #[target_feature(enable = "vector")]
3502                    unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self {
3503                        static_assert_uimm_bits!(C, 2);
3504                        transmute(vsldb(transmute(self), transmute(b), const { 4 * C }))
3505                    }
3506
3507                    #[inline]
3508                    #[target_feature(enable = "vector-enhancements-2")]
3509                    unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self {
3510                        static_assert_uimm_bits!(C, 3);
3511                        transmute(vsld(transmute(self), transmute(b), C))
3512                    }
3513                }
3514
3515                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3516                impl VectorSrdb for $ty {
3517                    #[inline]
3518                    #[target_feature(enable = "vector-enhancements-2")]
3519                    unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self {
3520                        static_assert_uimm_bits!(C, 3);
3521                        transmute(vsrd(transmute(self), transmute(b), C))
3522                    }
3523                }
3524            )*
3525        }
3526    }
3527
3528    impl_vec_sld! {
3529        vector_signed_char
3530        vector_bool_char
3531        vector_unsigned_char
3532
3533        vector_signed_short
3534        vector_bool_short
3535        vector_unsigned_short
3536
3537        vector_signed_int
3538        vector_bool_int
3539        vector_unsigned_int
3540
3541        vector_signed_long_long
3542        vector_bool_long_long
3543        vector_unsigned_long_long
3544
3545        vector_float
3546        vector_double
3547    }
3548
3549    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3550    pub trait VectorCompareRange: Sized {
3551        type Result;
3552
3553        unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3554        unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3555        unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3556        unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3557    }
3558
3559    const fn validate_compare_range_imm(imm: u32) {
3560        if !matches!(imm, 0 | 4 | 8 | 12) {
3561            panic!("IMM needs to be one of 0, 4, 8, 12");
3562        }
3563    }
3564
3565    macro_rules! impl_compare_range {
3566        ($($ty:ident $vstrc:ident $vstrcs:ident $vstrcz:ident $vstrczs:ident)*) => {
3567            $(
3568                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3569                impl VectorCompareRange for $ty {
3570                    type Result = t_b!($ty);
3571
3572                    #[inline]
3573                    #[target_feature(enable = "vector")]
3574                    unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3575                        const { validate_compare_range_imm };
3576                        $vstrc(self, b, c, IMM)
3577                    }
3578
3579                    #[inline]
3580                    #[target_feature(enable = "vector")]
3581                    unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3582                        const { validate_compare_range_imm };
3583                        $vstrcz(self, b, c, IMM)
3584                    }
3585
3586                    #[inline]
3587                    #[target_feature(enable = "vector")]
3588                    unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3589                        const { validate_compare_range_imm };
3590                        let PackedTuple { x, y } = $vstrcs(self, b, c, IMM);
3591                        (x,y)
3592                    }
3593
3594                    #[inline]
3595                    #[target_feature(enable = "vector")]
3596                    unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3597                        const { validate_compare_range_imm };
3598                        let PackedTuple { x, y } = $vstrczs(self, b, c, IMM);
3599                        (x,y)
3600                    }
3601                }
3602            )*
3603        }
3604    }
3605
3606    impl_compare_range! {
3607        vector_unsigned_char    vstrcb vstrcbs vstrczb vstrczbs
3608        vector_unsigned_short   vstrch vstrchs vstrczh vstrczhs
3609        vector_unsigned_int     vstrcf vstrcfs vstrczf vstrczfs
3610    }
3611
3612    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3613    pub trait VectorComparePredicate: Sized {
3614        type Result;
3615
3616        #[inline]
3617        #[target_feature(enable = "vector")]
3618        unsafe fn vec_cmpgt(self, other: Self) -> Self::Result {
3619            simd_gt(self, other)
3620        }
3621
3622        #[inline]
3623        #[target_feature(enable = "vector")]
3624        unsafe fn vec_cmpge(self, other: Self) -> Self::Result {
3625            simd_ge(self, other)
3626        }
3627
3628        #[inline]
3629        #[target_feature(enable = "vector")]
3630        unsafe fn vec_cmplt(self, other: Self) -> Self::Result {
3631            simd_lt(self, other)
3632        }
3633
3634        #[inline]
3635        #[target_feature(enable = "vector")]
3636        unsafe fn vec_cmple(self, other: Self) -> Self::Result {
3637            simd_le(self, other)
3638        }
3639    }
3640
3641    macro_rules! impl_compare_predicate {
3642        ($($ty:ident)*) => {
3643            $(
3644                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3645                impl VectorComparePredicate for $ty {
3646                    type Result = t_b!($ty);
3647                }
3648            )*
3649        }
3650    }
3651
3652    impl_compare_predicate! {
3653        vector_signed_char
3654        vector_unsigned_char
3655
3656        vector_signed_short
3657        vector_unsigned_short
3658
3659        vector_signed_int
3660        vector_unsigned_int
3661        vector_float
3662
3663        vector_signed_long_long
3664        vector_unsigned_long_long
3665        vector_double
3666    }
3667
3668    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3669    pub trait VectorEquality: Sized {
3670        type Result;
3671
3672        #[inline]
3673        #[target_feature(enable = "vector")]
3674        unsafe fn vec_cmpeq(self, other: Self) -> Self::Result {
3675            simd_eq(self, other)
3676        }
3677
3678        #[inline]
3679        #[target_feature(enable = "vector")]
3680        unsafe fn vec_cmpne(self, other: Self) -> Self::Result {
3681            simd_ne(self, other)
3682        }
3683    }
3684
3685    macro_rules! impl_compare_equality {
3686        ($($ty:ident)*) => {
3687            $(
3688                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3689                impl VectorEquality for $ty {
3690                    type Result = t_b!($ty);
3691                }
3692            )*
3693        }
3694    }
3695
3696    impl_compare_equality! {
3697        vector_bool_char
3698        vector_signed_char
3699        vector_unsigned_char
3700
3701        vector_bool_short
3702        vector_signed_short
3703        vector_unsigned_short
3704
3705        vector_bool_int
3706        vector_signed_int
3707        vector_unsigned_int
3708        vector_float
3709
3710        vector_bool_long_long
3711        vector_signed_long_long
3712        vector_unsigned_long_long
3713        vector_double
3714    }
3715
3716    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3717    pub trait VectorEqualityIdx: Sized {
3718        type Result;
3719
3720        unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result;
3721        unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result;
3722
3723        unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32);
3724        unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32);
3725
3726        unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result;
3727        unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result;
3728
3729        unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3730        unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3731    }
3732
3733    macro_rules! impl_compare_equality_idx {
3734        ($($ty:ident $ret:ident
3735                $cmpeq:ident $cmpne:ident
3736                $cmpeq_or_0:ident $cmpne_or_0:ident
3737                $cmpeq_cc:ident $cmpne_cc:ident
3738                $cmpeq_or_0_cc:ident $cmpne_or_0_cc:ident
3739        )*) => {
3740            $(
3741                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3742                impl VectorEqualityIdx for $ty {
3743                    type Result = $ret;
3744
3745                    #[inline]
3746                    #[target_feature(enable = "vector")]
3747                    unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result {
3748                        transmute($cmpeq(transmute(self), transmute(other)))
3749                    }
3750
3751                    #[inline]
3752                    #[target_feature(enable = "vector")]
3753                    unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result {
3754                        transmute($cmpne(transmute(self), transmute(other)))
3755                    }
3756
3757                    #[inline]
3758                    #[target_feature(enable = "vector")]
3759                    unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result {
3760                        transmute($cmpeq_or_0(transmute(self), transmute(other)))
3761                    }
3762
3763                    #[inline]
3764                    #[target_feature(enable = "vector")]
3765                    unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result {
3766                        transmute($cmpne_or_0(transmute(self), transmute(other)))
3767                    }
3768
3769                    #[inline]
3770                    #[target_feature(enable = "vector")]
3771                    unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32) {
3772                        let PackedTuple { x, y } = $cmpeq_cc(transmute(self), transmute(other));
3773                        (transmute(x), y)
3774                    }
3775
3776                    #[inline]
3777                    #[target_feature(enable = "vector")]
3778                    unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32) {
3779                        let PackedTuple { x, y } = $cmpne_cc(transmute(self), transmute(other));
3780                        (transmute(x),y)
3781                    }
3782
3783                    #[inline]
3784                    #[target_feature(enable = "vector")]
3785                    unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3786                        let PackedTuple { x, y } = $cmpeq_or_0_cc(transmute(self), transmute(other));
3787                        (transmute(x), y)
3788                    }
3789
3790                    #[inline]
3791                    #[target_feature(enable = "vector")]
3792                    unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3793                        let PackedTuple { x, y } = $cmpne_or_0_cc(transmute(self), transmute(other));
3794                        (transmute(x),y)
3795                    }
3796                }
3797            )*
3798        }
3799    }
3800
3801    impl_compare_equality_idx! {
3802        vector_signed_char vector_signed_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3803        vector_bool_char vector_unsigned_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3804        vector_unsigned_char vector_unsigned_char           vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3805        vector_signed_short vector_signed_short             vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3806        vector_bool_short  vector_unsigned_short            vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3807        vector_unsigned_short vector_unsigned_short         vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3808        vector_signed_int vector_signed_int                 vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3809        vector_bool_int  vector_unsigned_int                vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3810        vector_unsigned_int vector_unsigned_int             vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3811    }
3812}
3813
3814/// Load Count to Block Boundary
3815#[inline]
3816#[target_feature(enable = "vector")]
3817#[unstable(feature = "stdarch_s390x", issue = "135681")]
3818#[cfg_attr(test, assert_instr(lcbb, BLOCK_BOUNDARY = 512))]
3819unsafe fn __lcbb<const BLOCK_BOUNDARY: u16>(ptr: *const u8) -> u32 {
3820    lcbb(ptr, const { validate_block_boundary(BLOCK_BOUNDARY) })
3821}
3822
3823/// Vector Add
3824#[inline]
3825#[target_feature(enable = "vector")]
3826#[unstable(feature = "stdarch_s390x", issue = "135681")]
3827pub unsafe fn vec_add<T: sealed::VectorAdd<U>, U>(a: T, b: U) -> T::Result {
3828    a.vec_add(b)
3829}
3830
3831/// Vector Subtract
3832#[inline]
3833#[target_feature(enable = "vector")]
3834#[unstable(feature = "stdarch_s390x", issue = "135681")]
3835pub unsafe fn vec_sub<T: sealed::VectorSub<U>, U>(a: T, b: U) -> T::Result {
3836    a.vec_sub(b)
3837}
3838
3839/// Vector Multiply
3840///
3841/// ## Purpose
3842/// Compute the products of corresponding elements of two vectors.
3843///
3844/// ## Result value
3845/// Each element of r receives the product of the corresponding elements of a and b.
3846#[inline]
3847#[target_feature(enable = "vector")]
3848#[unstable(feature = "stdarch_s390x", issue = "135681")]
3849pub unsafe fn vec_mul<T: sealed::VectorMul>(a: T, b: T) -> T {
3850    a.vec_mul(b)
3851}
3852
3853/// Vector Count Leading Zeros
3854#[inline]
3855#[target_feature(enable = "vector")]
3856#[unstable(feature = "stdarch_s390x", issue = "135681")]
3857pub unsafe fn vec_cntlz<T: sealed::CountBits>(a: T) -> T::Result {
3858    a.vec_cntlz()
3859}
3860
3861/// Vector Count Trailing Zeros
3862#[inline]
3863#[target_feature(enable = "vector")]
3864#[unstable(feature = "stdarch_s390x", issue = "135681")]
3865pub unsafe fn vec_cnttz<T: sealed::CountBits>(a: T) -> T::Result {
3866    a.vec_cnttz()
3867}
3868
3869/// Vector Population Count
3870///
3871/// Computes the population count (number of set bits) in each element of the input.
3872#[inline]
3873#[target_feature(enable = "vector")]
3874#[unstable(feature = "stdarch_s390x", issue = "135681")]
3875pub unsafe fn vec_popcnt<T: sealed::CountBits>(a: T) -> T::Result {
3876    a.vec_popcnt()
3877}
3878
3879/// Vector Maximum
3880#[inline]
3881#[target_feature(enable = "vector")]
3882#[unstable(feature = "stdarch_s390x", issue = "135681")]
3883pub unsafe fn vec_max<T: sealed::VectorMax<U>, U>(a: T, b: U) -> T::Result {
3884    a.vec_max(b)
3885}
3886
3887/// Vector  Minimum
3888#[inline]
3889#[target_feature(enable = "vector")]
3890#[unstable(feature = "stdarch_s390x", issue = "135681")]
3891pub unsafe fn vec_min<T: sealed::VectorMin<U>, U>(a: T, b: U) -> T::Result {
3892    a.vec_min(b)
3893}
3894
3895/// Vector Absolute
3896#[inline]
3897#[target_feature(enable = "vector")]
3898#[unstable(feature = "stdarch_s390x", issue = "135681")]
3899pub unsafe fn vec_abs<T: sealed::VectorAbs>(a: T) -> T {
3900    a.vec_abs()
3901}
3902
3903/// Vector Negative Absolute
3904#[inline]
3905#[target_feature(enable = "vector")]
3906#[unstable(feature = "stdarch_s390x", issue = "135681")]
3907pub unsafe fn vec_nabs<T: sealed::VectorNabs>(a: T) -> T {
3908    a.vec_nabs()
3909}
3910
3911/// Vector Negative Multiply Add
3912#[inline]
3913#[target_feature(enable = "vector")]
3914#[unstable(feature = "stdarch_s390x", issue = "135681")]
3915pub unsafe fn vec_nmadd<T: sealed::VectorNmadd>(a: T, b: T, c: T) -> T {
3916    a.vec_nmadd(b, c)
3917}
3918
3919/// Vector Negative Multiply Subtract
3920#[inline]
3921#[target_feature(enable = "vector")]
3922#[unstable(feature = "stdarch_s390x", issue = "135681")]
3923pub unsafe fn vec_nmsub<T: sealed::VectorNmsub>(a: T, b: T, c: T) -> T {
3924    a.vec_nmsub(b, c)
3925}
3926
3927/// Vector Square Root
3928#[inline]
3929#[target_feature(enable = "vector")]
3930#[unstable(feature = "stdarch_s390x", issue = "135681")]
3931pub unsafe fn vec_sqrt<T: sealed::VectorSqrt>(a: T) -> T {
3932    a.vec_sqrt()
3933}
3934
3935/// Vector Splat
3936#[inline]
3937#[target_feature(enable = "vector")]
3938#[unstable(feature = "stdarch_s390x", issue = "135681")]
3939pub unsafe fn vec_splat<T: sealed::VectorSplat, const IMM: u32>(a: T) -> T {
3940    a.vec_splat::<IMM>()
3941}
3942
3943/// Vector Splats
3944#[inline]
3945#[target_feature(enable = "vector")]
3946#[unstable(feature = "stdarch_s390x", issue = "135681")]
3947pub unsafe fn vec_splats<T: sealed::VectorSplats<U>, U>(a: T) -> U {
3948    a.vec_splats()
3949}
3950
3951/// Vector AND
3952#[inline]
3953#[target_feature(enable = "vector")]
3954#[unstable(feature = "stdarch_s390x", issue = "135681")]
3955pub unsafe fn vec_and<T: sealed::VectorAnd<U>, U>(a: T, b: U) -> T::Result {
3956    a.vec_and(b)
3957}
3958
3959/// Vector OR
3960#[inline]
3961#[target_feature(enable = "vector")]
3962#[unstable(feature = "stdarch_s390x", issue = "135681")]
3963pub unsafe fn vec_or<T: sealed::VectorOr<U>, U>(a: T, b: U) -> T::Result {
3964    a.vec_or(b)
3965}
3966
3967/// Vector XOR
3968#[inline]
3969#[target_feature(enable = "vector")]
3970#[unstable(feature = "stdarch_s390x", issue = "135681")]
3971pub unsafe fn vec_xor<T: sealed::VectorXor<U>, U>(a: T, b: U) -> T::Result {
3972    a.vec_xor(b)
3973}
3974
3975/// Vector NOR
3976#[inline]
3977#[target_feature(enable = "vector")]
3978#[unstable(feature = "stdarch_s390x", issue = "135681")]
3979pub unsafe fn vec_nor<T: sealed::VectorNor<U>, U>(a: T, b: U) -> T::Result {
3980    a.vec_nor(b)
3981}
3982
3983/// Vector NAND
3984#[inline]
3985#[target_feature(enable = "vector")]
3986#[unstable(feature = "stdarch_s390x", issue = "135681")]
3987pub unsafe fn vec_nand<T: sealed::VectorNand<U>, U>(a: T, b: U) -> T::Result {
3988    a.vec_nand(b)
3989}
3990
3991/// Vector XNOR
3992#[inline]
3993#[target_feature(enable = "vector")]
3994#[unstable(feature = "stdarch_s390x", issue = "135681")]
3995pub unsafe fn vec_eqv<T: sealed::VectorEqv<U>, U>(a: T, b: U) -> T::Result {
3996    a.vec_eqv(b)
3997}
3998
3999/// Vector ANDC
4000#[inline]
4001#[target_feature(enable = "vector")]
4002#[unstable(feature = "stdarch_s390x", issue = "135681")]
4003pub unsafe fn vec_andc<T: sealed::VectorAndc<U>, U>(a: T, b: U) -> T::Result {
4004    a.vec_andc(b)
4005}
4006
4007/// Vector OR with Complement
4008///
4009/// ## Purpose
4010/// Performs a bitwise OR of the first vector with the bitwise-complemented second vector.
4011///
4012/// ## Result value
4013/// r is the bitwise OR of a and the bitwise complement of b.
4014#[inline]
4015#[target_feature(enable = "vector")]
4016#[unstable(feature = "stdarch_s390x", issue = "135681")]
4017pub unsafe fn vec_orc<T: sealed::VectorOrc<U>, U>(a: T, b: U) -> T::Result {
4018    a.vec_orc(b)
4019}
4020
4021/// Vector Floor
4022#[inline]
4023#[target_feature(enable = "vector")]
4024#[unstable(feature = "stdarch_s390x", issue = "135681")]
4025pub unsafe fn vec_floor<T: sealed::VectorFloor>(a: T) -> T {
4026    a.vec_floor()
4027}
4028
4029/// Vector Ceil
4030#[inline]
4031#[target_feature(enable = "vector")]
4032#[unstable(feature = "stdarch_s390x", issue = "135681")]
4033pub unsafe fn vec_ceil<T: sealed::VectorCeil>(a: T) -> T {
4034    a.vec_ceil()
4035}
4036
4037/// Vector Truncate
4038///
4039/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4040/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4041#[inline]
4042#[target_feature(enable = "vector")]
4043#[unstable(feature = "stdarch_s390x", issue = "135681")]
4044pub unsafe fn vec_trunc<T: sealed::VectorTrunc>(a: T) -> T {
4045    a.vec_trunc()
4046}
4047
4048/// Vector Round
4049///
4050/// Returns a vector containing the rounded values to the nearest representable floating-point integer,
4051/// using IEEE round-to-nearest rounding, of the corresponding elements of the given vector
4052#[inline]
4053#[target_feature(enable = "vector")]
4054#[unstable(feature = "stdarch_s390x", issue = "135681")]
4055pub unsafe fn vec_round<T: sealed::VectorRound>(a: T) -> T {
4056    a.vec_round()
4057}
4058
4059/// Vector Round to Current
4060///
4061/// Returns a vector by using the current rounding mode to round every
4062/// floating-point element in the given vector to integer.
4063#[inline]
4064#[target_feature(enable = "vector")]
4065#[unstable(feature = "stdarch_s390x", issue = "135681")]
4066pub unsafe fn vec_roundc<T: sealed::VectorRoundc>(a: T) -> T {
4067    a.vec_roundc()
4068}
4069
4070/// Vector Round toward Negative Infinity
4071///
4072/// Returns a vector containing the largest representable floating-point integral values less
4073/// than or equal to the values of the corresponding elements of the given vector.
4074#[inline]
4075#[target_feature(enable = "vector")]
4076#[unstable(feature = "stdarch_s390x", issue = "135681")]
4077pub unsafe fn vec_roundm<T: sealed::VectorFloor>(a: T) -> T {
4078    // the IBM docs note
4079    //
4080    // > vec_roundm provides the same functionality as vec_floor, except that vec_roundz would not trigger the IEEE-inexact exception.
4081    //
4082    // but in practice `vec_floor` also does not trigger that exception, so both are equivalent
4083    a.vec_floor()
4084}
4085
4086/// Vector Round toward Positive Infinity
4087///
4088/// Returns a vector containing the smallest representable floating-point integral values greater
4089/// than or equal to the values of the corresponding elements of the given vector.
4090#[inline]
4091#[target_feature(enable = "vector")]
4092#[unstable(feature = "stdarch_s390x", issue = "135681")]
4093pub unsafe fn vec_roundp<T: sealed::VectorCeil>(a: T) -> T {
4094    // the IBM docs note
4095    //
4096    // > vec_roundp provides the same functionality as vec_ceil, except that vec_roundz would not trigger the IEEE-inexact exception.
4097    //
4098    // but in practice `vec_ceil` also does not trigger that exception, so both are equivalent
4099    a.vec_ceil()
4100}
4101
4102/// Vector Round toward Zero
4103///
4104/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4105/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4106#[inline]
4107#[target_feature(enable = "vector")]
4108#[unstable(feature = "stdarch_s390x", issue = "135681")]
4109pub unsafe fn vec_roundz<T: sealed::VectorTrunc>(a: T) -> T {
4110    // the IBM docs note
4111    //
4112    // > vec_roundz provides the same functionality as vec_trunc, except that vec_roundz would not trigger the IEEE-inexact exception.
4113    //
4114    // but in practice `vec_trunc` also does not trigger that exception, so both are equivalent
4115    a.vec_trunc()
4116}
4117
4118/// Vector Round to Integer
4119///
4120/// Returns a vector by using the current rounding mode to round every floating-point element in the given vector to integer.
4121#[inline]
4122#[target_feature(enable = "vector")]
4123#[unstable(feature = "stdarch_s390x", issue = "135681")]
4124pub unsafe fn vec_rint<T: sealed::VectorRint>(a: T) -> T {
4125    a.vec_rint()
4126}
4127
4128/// Vector Average
4129#[inline]
4130#[target_feature(enable = "vector")]
4131#[unstable(feature = "stdarch_s390x", issue = "135681")]
4132pub unsafe fn vec_avg<T: sealed::VectorAvg<U>, U>(a: T, b: U) -> T::Result {
4133    a.vec_avg(b)
4134}
4135
4136/// Vector Shift Left
4137#[inline]
4138#[target_feature(enable = "vector")]
4139#[unstable(feature = "stdarch_s390x", issue = "135681")]
4140pub unsafe fn vec_sl<T: sealed::VectorSl<U>, U>(a: T, b: U) -> T::Result {
4141    a.vec_sl(b)
4142}
4143
4144/// Vector Shift Right
4145#[inline]
4146#[target_feature(enable = "vector")]
4147#[unstable(feature = "stdarch_s390x", issue = "135681")]
4148pub unsafe fn vec_sr<T: sealed::VectorSr<U>, U>(a: T, b: U) -> T::Result {
4149    a.vec_sr(b)
4150}
4151
4152/// Vector Shift Right Algebraic
4153#[inline]
4154#[target_feature(enable = "vector")]
4155#[unstable(feature = "stdarch_s390x", issue = "135681")]
4156pub unsafe fn vec_sra<T: sealed::VectorSra<U>, U>(a: T, b: U) -> T::Result {
4157    a.vec_sra(b)
4158}
4159
4160/// Vector Shift Left by Byte
4161#[inline]
4162#[target_feature(enable = "vector")]
4163#[unstable(feature = "stdarch_s390x", issue = "135681")]
4164pub unsafe fn vec_slb<T: sealed::VectorSlb<U>, U>(a: T, b: U) -> T::Result {
4165    a.vec_slb(b)
4166}
4167
4168/// Vector Shift Right by Byte
4169#[inline]
4170#[target_feature(enable = "vector")]
4171#[unstable(feature = "stdarch_s390x", issue = "135681")]
4172pub unsafe fn vec_srb<T: sealed::VectorSrb<U>, U>(a: T, b: U) -> T::Result {
4173    a.vec_srb(b)
4174}
4175
4176/// Vector Shift Right Algebraic by Byte
4177#[inline]
4178#[target_feature(enable = "vector")]
4179#[unstable(feature = "stdarch_s390x", issue = "135681")]
4180pub unsafe fn vec_srab<T: sealed::VectorSrab<U>, U>(a: T, b: U) -> T::Result {
4181    a.vec_srab(b)
4182}
4183
4184/// Vector Element Rotate Left
4185#[inline]
4186#[target_feature(enable = "vector")]
4187#[unstable(feature = "stdarch_s390x", issue = "135681")]
4188pub unsafe fn vec_rl<T: sealed::VectorRl<U>, U>(a: T, b: U) -> T::Result {
4189    a.vec_rl(b)
4190}
4191
4192/// Vector Shift Left
4193///
4194/// Performs a left shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4195/// element of a left by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4196#[inline]
4197#[target_feature(enable = "vector")]
4198#[unstable(feature = "stdarch_s390x", issue = "135681")]
4199pub unsafe fn vec_sll<T>(a: T, b: vector_unsigned_char) -> T
4200where
4201    T: sealed::VectorSll<vector_unsigned_char, Result = T>,
4202{
4203    a.vec_sll(b)
4204}
4205
4206/// Vector Shift Right
4207///
4208/// Performs a right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4209/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4210#[inline]
4211#[target_feature(enable = "vector")]
4212#[unstable(feature = "stdarch_s390x", issue = "135681")]
4213pub unsafe fn vec_srl<T>(a: T, b: vector_unsigned_char) -> T
4214where
4215    T: sealed::VectorSrl<vector_unsigned_char, Result = T>,
4216{
4217    a.vec_srl(b)
4218}
4219
4220/// Vector Shift Right Arithmetic
4221///
4222/// Performs an algebraic right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4223/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by copies of
4224/// the most significant bit of the element of a.
4225#[inline]
4226#[target_feature(enable = "vector")]
4227#[unstable(feature = "stdarch_s390x", issue = "135681")]
4228pub unsafe fn vec_sral<T>(a: T, b: vector_unsigned_char) -> T
4229where
4230    T: sealed::VectorSral<vector_unsigned_char, Result = T>,
4231{
4232    a.vec_sral(b)
4233}
4234
4235/// Vector Element Rotate Left Immediate
4236///
4237/// Rotates each element of a vector left by a given number of bits. Each element of the result is obtained by rotating the corresponding element
4238/// of a left by the number of bits specified by b, modulo the number of bits in the element.
4239#[inline]
4240#[target_feature(enable = "vector")]
4241#[unstable(feature = "stdarch_s390x", issue = "135681")]
4242pub unsafe fn vec_rli<T: sealed::VectorRli>(a: T, bits: core::ffi::c_ulong) -> T {
4243    a.vec_rli(bits)
4244}
4245
4246/// Vector Reverse Elements
4247///
4248/// Returns a vector with the elements of the input vector in reversed order.
4249#[inline]
4250#[target_feature(enable = "vector")]
4251#[unstable(feature = "stdarch_s390x", issue = "135681")]
4252pub unsafe fn vec_reve<T: sealed::VectorReve>(a: T) -> T {
4253    a.vec_reve()
4254}
4255
4256/// Vector Byte Reverse
4257///
4258/// Returns a vector where each vector element contains the corresponding byte-reversed vector element of the input vector.
4259#[inline]
4260#[target_feature(enable = "vector")]
4261#[unstable(feature = "stdarch_s390x", issue = "135681")]
4262pub unsafe fn vec_revb<T: sealed::VectorRevb>(a: T) -> T {
4263    a.vec_revb()
4264}
4265
4266/// Vector Merge High
4267///
4268/// Merges the most significant ("high") halves of two vectors.
4269#[inline]
4270#[target_feature(enable = "vector")]
4271#[unstable(feature = "stdarch_s390x", issue = "135681")]
4272pub unsafe fn vec_mergeh<T: sealed::VectorMergeh>(a: T, b: T) -> T {
4273    a.vec_mergeh(b)
4274}
4275
4276/// Vector Merge Low
4277///
4278/// Merges the least significant ("low") halves of two vectors.
4279#[inline]
4280#[target_feature(enable = "vector")]
4281#[unstable(feature = "stdarch_s390x", issue = "135681")]
4282pub unsafe fn vec_mergel<T: sealed::VectorMergel>(a: T, b: T) -> T {
4283    a.vec_mergel(b)
4284}
4285
4286/// Vector Pack
4287#[inline]
4288#[target_feature(enable = "vector")]
4289#[unstable(feature = "stdarch_s390x", issue = "135681")]
4290pub unsafe fn vec_pack<T: sealed::VectorPack<U>, U>(a: T, b: U) -> T::Result {
4291    a.vec_pack(b)
4292}
4293
4294/// Vector Pack Saturated
4295#[inline]
4296#[target_feature(enable = "vector")]
4297#[unstable(feature = "stdarch_s390x", issue = "135681")]
4298pub unsafe fn vec_packs<T: sealed::VectorPacks<U>, U>(a: T, b: U) -> T::Result {
4299    a.vec_packs(b)
4300}
4301
4302/// Vector Pack Saturated Condition Code
4303#[inline]
4304#[target_feature(enable = "vector")]
4305#[unstable(feature = "stdarch_s390x", issue = "135681")]
4306pub unsafe fn vec_packs_cc<T: sealed::VectorPacksCC>(a: T, b: T, c: *mut i32) -> T::Result {
4307    let (x, y) = a.vec_packs_cc(b);
4308    unsafe { c.write(y) };
4309    x
4310}
4311
4312/// Vector Pack Saturated Unsigned
4313#[inline]
4314#[target_feature(enable = "vector")]
4315#[unstable(feature = "stdarch_s390x", issue = "135681")]
4316pub unsafe fn vec_packsu<T: sealed::VectorPacksu<U>, U>(a: T, b: U) -> T::Result {
4317    a.vec_packsu(b)
4318}
4319
4320/// Vector Pack Saturated Unsigned Condition Code
4321#[inline]
4322#[target_feature(enable = "vector")]
4323#[unstable(feature = "stdarch_s390x", issue = "135681")]
4324pub unsafe fn vec_packsu_cc<T: sealed::VectorPacksuCC>(a: T, b: T, c: *mut i32) -> T::Result {
4325    let (x, y) = a.vec_packsu_cc(b);
4326    unsafe { c.write(y) };
4327    x
4328}
4329
4330/// Vector Unpack High
4331#[inline]
4332#[target_feature(enable = "vector")]
4333#[unstable(feature = "stdarch_s390x", issue = "135681")]
4334pub unsafe fn vec_unpackh<T: sealed::VectorUnpackh>(a: T) -> <T as sealed::VectorUnpackh>::Result {
4335    a.vec_unpackh()
4336}
4337
4338/// Vector Unpack Low
4339#[inline]
4340#[target_feature(enable = "vector")]
4341#[unstable(feature = "stdarch_s390x", issue = "135681")]
4342pub unsafe fn vec_unpackl<T: sealed::VectorUnpackl>(a: T) -> <T as sealed::VectorUnpackl>::Result {
4343    a.vec_unpackl()
4344}
4345
4346/// Vector Generate Byte Mask
4347///
4348/// Generates byte masks for elements in the return vector. For each bit in a, if the bit is one, all bit positions
4349/// in the corresponding byte element of d are set to ones. Otherwise, if the bit is zero, the corresponding byte element is set to zero.
4350#[inline]
4351#[target_feature(enable = "vector")]
4352#[unstable(feature = "stdarch_s390x", issue = "135681")]
4353#[cfg_attr(test, assert_instr(vgbm, MASK = 0x00FF))]
4354pub unsafe fn vec_genmask<const MASK: u16>() -> vector_unsigned_char {
4355    vector_unsigned_char(const { genmask::<MASK>() })
4356}
4357
4358/// Vector Generate Mask (Byte)
4359#[inline]
4360#[target_feature(enable = "vector")]
4361#[unstable(feature = "stdarch_s390x", issue = "135681")]
4362#[cfg_attr(test, assert_instr(vrepib, L = 3, H = 5))]
4363pub unsafe fn vec_genmasks_8<const L: u8, const H: u8>() -> vector_unsigned_char {
4364    vector_unsigned_char(const { [genmasks(u8::BITS, L, H) as u8; 16] })
4365}
4366
4367/// Vector Generate Mask (Halfword)
4368#[inline]
4369#[target_feature(enable = "vector")]
4370#[unstable(feature = "stdarch_s390x", issue = "135681")]
4371#[cfg_attr(test, assert_instr(vrepih, L = 3, H = 5))]
4372pub unsafe fn vec_genmasks_16<const L: u8, const H: u8>() -> vector_unsigned_short {
4373    vector_unsigned_short(const { [genmasks(u16::BITS, L, H) as u16; 8] })
4374}
4375
4376/// Vector Generate Mask (Word)
4377#[inline]
4378#[target_feature(enable = "vector")]
4379#[unstable(feature = "stdarch_s390x", issue = "135681")]
4380#[cfg_attr(test, assert_instr(vgmf, L = 3, H = 5))]
4381pub unsafe fn vec_genmasks_32<const L: u8, const H: u8>() -> vector_unsigned_int {
4382    vector_unsigned_int(const { [genmasks(u32::BITS, L, H) as u32; 4] })
4383}
4384
4385/// Vector Generate Mask (Doubleword)
4386#[inline]
4387#[target_feature(enable = "vector")]
4388#[unstable(feature = "stdarch_s390x", issue = "135681")]
4389#[cfg_attr(test, assert_instr(vgmg, L = 3, H = 5))]
4390pub unsafe fn vec_genmasks_64<const L: u8, const H: u8>() -> vector_unsigned_long_long {
4391    vector_unsigned_long_long(const { [genmasks(u64::BITS, L, H); 2] })
4392}
4393
4394/// Vector Permute
4395///
4396/// Returns a vector that contains some elements of two vectors, in the order specified by a third vector.
4397/// Each byte of the result is selected by using the least significant 5 bits of the corresponding byte of c as an index into the concatenated bytes of a and b.
4398/// Note: The vector generate mask built-in function [`vec_genmask`] could help generate the mask c.
4399#[inline]
4400#[target_feature(enable = "vector")]
4401#[unstable(feature = "stdarch_s390x", issue = "135681")]
4402pub unsafe fn vec_perm<T: sealed::VectorPerm>(a: T, b: T, c: vector_unsigned_char) -> T {
4403    a.vec_perm(b, c)
4404}
4405
4406/// Vector Sum Across Quadword
4407///
4408/// Returns a vector containing the results of performing a sum across all the elements in each of the quadword of vector a,
4409/// and the rightmost word or doubleword element of the b. The result is an unsigned 128-bit integer.
4410#[inline]
4411#[target_feature(enable = "vector")]
4412#[unstable(feature = "stdarch_s390x", issue = "135681")]
4413pub unsafe fn vec_sum_u128<T: sealed::VectorSumU128>(a: T, b: T) -> vector_unsigned_char {
4414    a.vec_sum_u128(b)
4415}
4416
4417/// Vector Sum Across Doubleword
4418///
4419/// Returns a vector containing the results of performing a sum across all the elements in each of the doubleword of vector a,
4420/// and the rightmost sub-element of the corresponding doubleword of b.
4421#[inline]
4422#[target_feature(enable = "vector")]
4423#[unstable(feature = "stdarch_s390x", issue = "135681")]
4424pub unsafe fn vec_sum2<T: sealed::VectorSum2>(a: T, b: T) -> vector_unsigned_long_long {
4425    a.vec_sum2(b)
4426}
4427
4428/// Vector Sum Across Word
4429///
4430/// Returns a vector containing the results of performing a sum across all the elements in each of the word of vector a,
4431/// and the rightmost sub-element of the corresponding word of b.
4432#[inline]
4433#[target_feature(enable = "vector")]
4434#[unstable(feature = "stdarch_s390x", issue = "135681")]
4435pub unsafe fn vec_sum4<T: sealed::VectorSum4>(a: T, b: T) -> vector_unsigned_int {
4436    a.vec_sum4(b)
4437}
4438
4439/// Vector Addition unsigned 128-bits
4440///
4441/// Adds unsigned quadword values.
4442///
4443/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a + b.
4444#[inline]
4445#[target_feature(enable = "vector")]
4446#[unstable(feature = "stdarch_s390x", issue = "135681")]
4447#[cfg_attr(test, assert_instr(vaq))]
4448pub unsafe fn vec_add_u128(
4449    a: vector_unsigned_char,
4450    b: vector_unsigned_char,
4451) -> vector_unsigned_char {
4452    let a: u128 = transmute(a);
4453    let b: u128 = transmute(b);
4454    transmute(a.wrapping_add(b))
4455}
4456
4457/// Vector Subtract unsigned 128-bits
4458///
4459/// Subtracts unsigned quadword values.
4460///
4461/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a - b.
4462#[inline]
4463#[target_feature(enable = "vector")]
4464#[unstable(feature = "stdarch_s390x", issue = "135681")]
4465#[cfg_attr(test, assert_instr(vsq))]
4466pub unsafe fn vec_sub_u128(
4467    a: vector_unsigned_char,
4468    b: vector_unsigned_char,
4469) -> vector_unsigned_char {
4470    let a: u128 = transmute(a);
4471    let b: u128 = transmute(b);
4472
4473    transmute(a.wrapping_sub(b))
4474}
4475
4476/// Vector Subtract Carryout
4477///
4478/// Returns a vector containing the borrow produced by subtracting each of corresponding elements of b from a.
4479///
4480/// On each resulting element, the value is 0 if a borrow occurred, or 1 if no borrow occurred.
4481#[inline]
4482#[target_feature(enable = "vector")]
4483#[unstable(feature = "stdarch_s390x", issue = "135681")]
4484pub unsafe fn vec_subc<T: sealed::VectorSubc<U>, U>(a: T, b: U) -> T::Result {
4485    a.vec_subc(b)
4486}
4487
4488/// Vector Subtract Carryout unsigned 128-bits
4489///
4490/// Gets the carry bit of the 128-bit subtraction of two quadword values.
4491/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the borrow produced by subtracting b from a, as unsigned 128-bits integers.
4492/// If no borrow occurred, the bit 127 of d is 1; otherwise it is set to 0. All other bits of d are 0.
4493#[inline]
4494#[target_feature(enable = "vector")]
4495#[unstable(feature = "stdarch_s390x", issue = "135681")]
4496#[cfg_attr(test, assert_instr(vscbiq))]
4497pub unsafe fn vec_subc_u128(
4498    a: vector_unsigned_char,
4499    b: vector_unsigned_char,
4500) -> vector_unsigned_char {
4501    // FIXME(llvm) sadly this does not work https://github.com/llvm/llvm-project/issues/129608
4502    // let a: u128 = transmute(a);
4503    // let b: u128 = transmute(b);
4504    // transmute(!a.overflowing_sub(b).1 as u128)
4505    transmute(vscbiq(transmute(a), transmute(b)))
4506}
4507
4508/// Vector Add Compute Carryout unsigned 128-bits
4509#[inline]
4510#[target_feature(enable = "vector")]
4511#[unstable(feature = "stdarch_s390x", issue = "135681")]
4512#[cfg_attr(test, assert_instr(vaccq))]
4513pub unsafe fn vec_addc_u128(
4514    a: vector_unsigned_char,
4515    b: vector_unsigned_char,
4516) -> vector_unsigned_char {
4517    let a: u128 = transmute(a);
4518    let b: u128 = transmute(b);
4519    transmute(a.overflowing_add(b).1 as u128)
4520}
4521
4522/// Vector Add With Carry unsigned 128-bits
4523#[inline]
4524#[target_feature(enable = "vector")]
4525#[unstable(feature = "stdarch_s390x", issue = "135681")]
4526#[cfg_attr(test, assert_instr(vacq))]
4527pub unsafe fn vec_adde_u128(
4528    a: vector_unsigned_char,
4529    b: vector_unsigned_char,
4530    c: vector_unsigned_char,
4531) -> vector_unsigned_char {
4532    let a: u128 = transmute(a);
4533    let b: u128 = transmute(b);
4534    let c: u128 = transmute(c);
4535    // FIXME(llvm) sadly this does not work
4536    //     let (d, _carry) = a.carrying_add(b, c & 1 != 0);
4537    //     transmute(d)
4538    transmute(vacq(a, b, c))
4539}
4540
4541/// Vector Add With Carry Compute Carry unsigned 128-bits
4542#[inline]
4543#[target_feature(enable = "vector")]
4544#[unstable(feature = "stdarch_s390x", issue = "135681")]
4545#[cfg_attr(test, assert_instr(vacccq))]
4546pub unsafe fn vec_addec_u128(
4547    a: vector_unsigned_char,
4548    b: vector_unsigned_char,
4549    c: vector_unsigned_char,
4550) -> vector_unsigned_char {
4551    let a: u128 = transmute(a);
4552    let b: u128 = transmute(b);
4553    let c: u128 = transmute(c);
4554    let (_d, carry) = a.carrying_add(b, c & 1 != 0);
4555    transmute(carry as u128)
4556}
4557
4558/// Vector Subtract with Carryout
4559///
4560/// Subtracts unsigned quadword values with carry bit from a previous operation.
4561///
4562/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the result of subtracting of b from a,
4563/// and the carryout bit from a previous operation.
4564///
4565/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4566#[inline]
4567#[target_feature(enable = "vector")]
4568#[unstable(feature = "stdarch_s390x", issue = "135681")]
4569#[cfg_attr(test, assert_instr(vsbiq))]
4570pub unsafe fn vec_sube_u128(
4571    a: vector_unsigned_char,
4572    b: vector_unsigned_char,
4573    c: vector_unsigned_char,
4574) -> vector_unsigned_char {
4575    transmute(vsbiq(transmute(a), transmute(b), transmute(c)))
4576}
4577
4578/// Vector Subtract with Carryout, Carryout
4579///
4580/// Gets the carry bit of the 128-bit subtraction of two quadword values with carry bit from the previous operation.
4581///
4582/// It returns a vector containing the carryout produced from the result of subtracting of b from a,
4583/// and the carryout bit from a previous operation. If no borrow occurred, the 127-bit of d is 1, otherwise 0.
4584/// All other bits of d are 0.
4585///
4586/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4587#[inline]
4588#[target_feature(enable = "vector")]
4589#[unstable(feature = "stdarch_s390x", issue = "135681")]
4590#[cfg_attr(test, assert_instr(vsbcbiq))]
4591pub unsafe fn vec_subec_u128(
4592    a: vector_unsigned_char,
4593    b: vector_unsigned_char,
4594    c: vector_unsigned_char,
4595) -> vector_unsigned_char {
4596    transmute(vsbcbiq(transmute(a), transmute(b), transmute(c)))
4597}
4598
4599/// Vector Splat Signed Byte
4600#[inline]
4601#[target_feature(enable = "vector")]
4602#[unstable(feature = "stdarch_s390x", issue = "135681")]
4603#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4604pub unsafe fn vec_splat_s8<const IMM: i8>() -> vector_signed_char {
4605    vector_signed_char([IMM; 16])
4606}
4607
4608/// Vector Splat Signed Halfword
4609#[inline]
4610#[target_feature(enable = "vector")]
4611#[unstable(feature = "stdarch_s390x", issue = "135681")]
4612#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4613pub unsafe fn vec_splat_s16<const IMM: i16>() -> vector_signed_short {
4614    vector_signed_short([IMM as i16; 8])
4615}
4616
4617/// Vector Splat Signed Word
4618#[inline]
4619#[target_feature(enable = "vector")]
4620#[unstable(feature = "stdarch_s390x", issue = "135681")]
4621#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4622pub unsafe fn vec_splat_s32<const IMM: i16>() -> vector_signed_int {
4623    vector_signed_int([IMM as i32; 4])
4624}
4625
4626/// Vector Splat Signed Doubleword
4627#[inline]
4628#[target_feature(enable = "vector")]
4629#[unstable(feature = "stdarch_s390x", issue = "135681")]
4630#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4631pub unsafe fn vec_splat_s64<const IMM: i16>() -> vector_signed_long_long {
4632    vector_signed_long_long([IMM as i64; 2])
4633}
4634
4635/// Vector Splat Unsigned Byte
4636#[inline]
4637#[target_feature(enable = "vector")]
4638#[unstable(feature = "stdarch_s390x", issue = "135681")]
4639#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4640pub unsafe fn vec_splat_u8<const IMM: u8>() -> vector_unsigned_char {
4641    vector_unsigned_char([IMM; 16])
4642}
4643
4644/// Vector Splat Unsigned Halfword
4645#[inline]
4646#[target_feature(enable = "vector")]
4647#[unstable(feature = "stdarch_s390x", issue = "135681")]
4648#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4649pub unsafe fn vec_splat_u16<const IMM: i16>() -> vector_unsigned_short {
4650    vector_unsigned_short([IMM as u16; 8])
4651}
4652
4653/// Vector Splat Unsigned Word
4654#[inline]
4655#[target_feature(enable = "vector")]
4656#[unstable(feature = "stdarch_s390x", issue = "135681")]
4657#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4658pub unsafe fn vec_splat_u32<const IMM: i16>() -> vector_unsigned_int {
4659    vector_unsigned_int([IMM as u32; 4])
4660}
4661
4662/// Vector Splat Unsigned Doubleword
4663#[inline]
4664#[target_feature(enable = "vector")]
4665#[unstable(feature = "stdarch_s390x", issue = "135681")]
4666#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4667pub unsafe fn vec_splat_u64<const IMM: i16>() -> vector_unsigned_long_long {
4668    vector_unsigned_long_long([IMM as u64; 2])
4669}
4670
4671macro_rules! vec_find_any {
4672    ($($Trait:ident $fun:ident $doc:literal)*) => {
4673        $(
4674            #[inline]
4675            #[target_feature(enable = "vector")]
4676            #[unstable(feature = "stdarch_s390x", issue = "135681")]
4677            #[doc = $doc]
4678            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> T::Result {
4679                a.$fun(b)
4680            }
4681        )*
4682    }
4683}
4684
4685vec_find_any! {
4686    VectorFindAnyEq vec_find_any_eq "Vector Find Any Element Equal with Condition Code"
4687    VectorFindAnyNe vec_find_any_ne "Vector Find Any Element Not Equal with Condition Code"
4688    VectorFindAnyEqIdx vec_find_any_eq_idx "Vector Find Any Element Equal Index with Condition Code"
4689    VectorFindAnyNeIdx vec_find_any_ne_idx "Vector Find Any Element Not Equal Index with Condition Code"
4690    VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx "Vector Find Any Element Equal or Zero Index with Condition Code"
4691    VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx "Vector Find Any Element Not Equal or Zero Index with Condition Code"
4692}
4693
4694macro_rules! vec_find_any_cc {
4695    ($($Trait:ident $fun:ident $doc:literal)*) => {
4696        $(
4697            #[inline]
4698            #[target_feature(enable = "vector")]
4699            #[unstable(feature = "stdarch_s390x", issue = "135681")]
4700            #[doc = $doc]
4701            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U, c: *mut i32) -> T::Result {
4702                let (x, y) = a.$fun(b);
4703                unsafe { c.write(y) };
4704                x
4705            }
4706        )*
4707    }
4708}
4709
4710vec_find_any_cc! {
4711    VectorFindAnyEqCC vec_find_any_eq_cc "Vector Find Any Element Equal with Condition Code"
4712    VectorFindAnyNeCC vec_find_any_ne_cc "Vector Find Any Element Not Equal with Condition Code"
4713    VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc "Vector Find Any Element Equal Index with Condition Code"
4714    VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc "Vector Find Any Element Not Equal Index with Condition Code"
4715    VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc "Vector Find Any Element Equal or Zero Index with Condition Code"
4716    VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc "Vector Find Any Element Not Equal or Zero Index with Condition Code"
4717}
4718
4719/// Vector Load
4720#[inline]
4721#[target_feature(enable = "vector")]
4722#[unstable(feature = "stdarch_s390x", issue = "135681")]
4723pub unsafe fn vec_xl<T: sealed::VectorLoad>(offset: isize, ptr: *const T::ElementType) -> T {
4724    T::vec_xl(offset, ptr)
4725}
4726
4727/// Vector Load Pair
4728#[inline]
4729#[target_feature(enable = "vector")]
4730#[unstable(feature = "stdarch_s390x", issue = "135681")]
4731pub unsafe fn vec_load_pair<T: sealed::VectorLoadPair>(a: T::ElementType, b: T::ElementType) -> T {
4732    T::vec_load_pair(a, b)
4733}
4734
4735/// Vector Load to Block Boundary
4736#[inline]
4737#[target_feature(enable = "vector")]
4738#[unstable(feature = "stdarch_s390x", issue = "135681")]
4739pub unsafe fn vec_load_bndry<T: sealed::VectorLoad, const BLOCK_BOUNDARY: u16>(
4740    ptr: *const T::ElementType,
4741) -> MaybeUninit<T> {
4742    T::vec_load_bndry::<BLOCK_BOUNDARY>(ptr)
4743}
4744
4745/// Vector Store
4746#[inline]
4747#[target_feature(enable = "vector")]
4748#[unstable(feature = "stdarch_s390x", issue = "135681")]
4749pub unsafe fn vec_xst<T: sealed::VectorStore>(vector: T, offset: isize, ptr: *mut T::ElementType) {
4750    vector.vec_xst(offset, ptr)
4751}
4752
4753/// Vector Load with Length
4754#[inline]
4755#[target_feature(enable = "vector")]
4756#[unstable(feature = "stdarch_s390x", issue = "135681")]
4757pub unsafe fn vec_load_len<T: sealed::VectorLoad>(
4758    ptr: *const T::ElementType,
4759    byte_count: u32,
4760) -> T {
4761    T::vec_load_len(ptr, byte_count)
4762}
4763
4764/// Vector Store with Length
4765#[inline]
4766#[target_feature(enable = "vector")]
4767#[unstable(feature = "stdarch_s390x", issue = "135681")]
4768pub unsafe fn vec_store_len<T: sealed::VectorStore>(
4769    vector: T,
4770    ptr: *mut T::ElementType,
4771    byte_count: u32,
4772) {
4773    vector.vec_store_len(ptr, byte_count)
4774}
4775
4776/// Vector Load Rightmost with Length
4777#[inline]
4778#[target_feature(enable = "vector-packed-decimal")]
4779#[unstable(feature = "stdarch_s390x", issue = "135681")]
4780#[cfg_attr(test, assert_instr(vlrlr))]
4781pub unsafe fn vec_load_len_r(ptr: *const u8, byte_count: u32) -> vector_unsigned_char {
4782    vlrl(byte_count, ptr)
4783}
4784
4785/// Vector Store Rightmost with Length
4786#[inline]
4787#[target_feature(enable = "vector-packed-decimal")]
4788#[unstable(feature = "stdarch_s390x", issue = "135681")]
4789#[cfg_attr(test, assert_instr(vstrlr))]
4790pub unsafe fn vec_store_len_r(vector: vector_unsigned_char, ptr: *mut u8, byte_count: u32) {
4791    vstrl(vector, byte_count, ptr)
4792}
4793
4794/// Vector Multiply Add
4795#[inline]
4796#[target_feature(enable = "vector-packed-decimal")]
4797#[unstable(feature = "stdarch_s390x", issue = "135681")]
4798pub unsafe fn vec_madd<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
4799    a.vec_madd(b, c)
4800}
4801
4802/// Vector Multiply Add
4803#[inline]
4804#[target_feature(enable = "vector-packed-decimal")]
4805#[unstable(feature = "stdarch_s390x", issue = "135681")]
4806pub unsafe fn vec_msub<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
4807    a.vec_msub(b, c)
4808}
4809
4810/// Vector Multiply and Add Even
4811#[inline]
4812#[target_feature(enable = "vector-packed-decimal")]
4813#[unstable(feature = "stdarch_s390x", issue = "135681")]
4814pub unsafe fn vec_meadd<T: sealed::VectorMeadd>(a: T, b: T, c: T::Result) -> T::Result {
4815    a.vec_meadd(b, c)
4816}
4817
4818/// Vector Multiply and Add Odd
4819#[inline]
4820#[target_feature(enable = "vector-packed-decimal")]
4821#[unstable(feature = "stdarch_s390x", issue = "135681")]
4822pub unsafe fn vec_moadd<T: sealed::VectorMoadd>(a: T, b: T, c: T::Result) -> T::Result {
4823    a.vec_moadd(b, c)
4824}
4825
4826/// Vector Multiply and Add High
4827#[inline]
4828#[target_feature(enable = "vector-packed-decimal")]
4829#[unstable(feature = "stdarch_s390x", issue = "135681")]
4830pub unsafe fn vec_mhadd<T: sealed::VectorMhadd>(a: T, b: T, c: T::Result) -> T::Result {
4831    a.vec_mhadd(b, c)
4832}
4833
4834/// Vector Multiply and Add Low
4835#[inline]
4836#[target_feature(enable = "vector-packed-decimal")]
4837#[unstable(feature = "stdarch_s390x", issue = "135681")]
4838pub unsafe fn vec_mladd<T: sealed::VectorMladd>(a: T, b: T, c: T::Result) -> T::Result {
4839    a.vec_mladd(b, c)
4840}
4841
4842/// Vector Checksum
4843#[inline]
4844#[target_feature(enable = "vector")]
4845#[unstable(feature = "stdarch_s390x", issue = "135681")]
4846#[cfg_attr(test, assert_instr(vcksm))]
4847pub unsafe fn vec_checksum(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int {
4848    vcksm(a, b)
4849}
4850
4851/// Vector Multiply Even
4852#[inline]
4853#[target_feature(enable = "vector")]
4854#[unstable(feature = "stdarch_s390x", issue = "135681")]
4855pub unsafe fn vec_mule<T: sealed::VectorMule<U>, U>(a: T, b: T) -> U {
4856    a.vec_mule(b)
4857}
4858
4859/// Vector Multiply Odd
4860#[inline]
4861#[target_feature(enable = "vector")]
4862#[unstable(feature = "stdarch_s390x", issue = "135681")]
4863pub unsafe fn vec_mulo<T: sealed::VectorMulo<U>, U>(a: T, b: T) -> U {
4864    a.vec_mulo(b)
4865}
4866
4867/// Vector Multiply High
4868#[inline]
4869#[target_feature(enable = "vector")]
4870#[unstable(feature = "stdarch_s390x", issue = "135681")]
4871pub unsafe fn vec_mulh<T: sealed::VectorMulh<U>, U>(a: T, b: T) -> U {
4872    a.vec_mulh(b)
4873}
4874
4875/// Vector Galois Field Multiply Sum
4876#[inline]
4877#[target_feature(enable = "vector")]
4878#[unstable(feature = "stdarch_s390x", issue = "135681")]
4879pub unsafe fn vec_gfmsum<T: sealed::VectorGfmsum<U>, U>(a: T, b: T) -> U {
4880    a.vec_gfmsum(b)
4881}
4882
4883/// Vector Galois Field Multiply Sum
4884#[inline]
4885#[target_feature(enable = "vector")]
4886#[unstable(feature = "stdarch_s390x", issue = "135681")]
4887pub unsafe fn vec_gfmsum_accum<T: sealed::VectorGfmsumAccum>(
4888    a: T,
4889    b: T,
4890    c: T::Result,
4891) -> T::Result {
4892    a.vec_gfmsum_accum(b, c)
4893}
4894
4895/// Vector Galois Field Multiply Sum 128-bits
4896#[inline]
4897#[target_feature(enable = "vector")]
4898#[unstable(feature = "stdarch_s390x", issue = "135681")]
4899#[cfg_attr(test, assert_instr(vgfmg))]
4900pub unsafe fn vec_gfmsum_128(
4901    a: vector_unsigned_long_long,
4902    b: vector_unsigned_long_long,
4903) -> vector_unsigned_char {
4904    transmute(vgfmg(a, b))
4905}
4906
4907/// Vector Galois Field Multiply Sum and Accumulate 128-bits
4908#[inline]
4909#[target_feature(enable = "vector")]
4910#[unstable(feature = "stdarch_s390x", issue = "135681")]
4911#[cfg_attr(test, assert_instr(vgfmag))]
4912pub unsafe fn vec_gfmsum_accum_128(
4913    a: vector_unsigned_long_long,
4914    b: vector_unsigned_long_long,
4915    c: vector_unsigned_char,
4916) -> vector_unsigned_char {
4917    transmute(vgfmag(a, b, transmute(c)))
4918}
4919
4920/// Vector Bit Permute
4921#[inline]
4922#[target_feature(enable = "vector-enhancements-1")]
4923#[unstable(feature = "stdarch_s390x", issue = "135681")]
4924#[cfg_attr(test, assert_instr(vbperm))]
4925pub unsafe fn vec_bperm_u128(
4926    a: vector_unsigned_char,
4927    b: vector_unsigned_char,
4928) -> vector_unsigned_long_long {
4929    vbperm(a, b)
4930}
4931
4932/// Vector Gather Element
4933#[inline]
4934#[target_feature(enable = "vector")]
4935#[unstable(feature = "stdarch_s390x", issue = "135681")]
4936pub unsafe fn vec_gather_element<T: sealed::VectorGatherElement, const D: u32>(
4937    a: T,
4938    b: T::Offset,
4939    c: *const T::Element,
4940) -> T {
4941    a.vec_gather_element::<D>(b, c)
4942}
4943
4944/// Vector Select
4945#[inline]
4946#[target_feature(enable = "vector")]
4947#[unstable(feature = "stdarch_s390x", issue = "135681")]
4948pub unsafe fn vec_sel<T: sealed::VectorSel<U>, U>(a: T, b: T, c: U) -> T {
4949    a.vec_sel(b, c)
4950}
4951
4952#[unstable(feature = "stdarch_s390x", issue = "135681")]
4953pub const __VEC_CLASS_FP_ZERO_P: u32 = 1 << 11;
4954#[unstable(feature = "stdarch_s390x", issue = "135681")]
4955pub const __VEC_CLASS_FP_ZERO_N: u32 = 1 << 10;
4956#[unstable(feature = "stdarch_s390x", issue = "135681")]
4957pub const __VEC_CLASS_FP_ZERO: u32 = __VEC_CLASS_FP_ZERO_P | __VEC_CLASS_FP_ZERO_N;
4958#[unstable(feature = "stdarch_s390x", issue = "135681")]
4959pub const __VEC_CLASS_FP_NORMAL_P: u32 = 1 << 9;
4960#[unstable(feature = "stdarch_s390x", issue = "135681")]
4961pub const __VEC_CLASS_FP_NORMAL_N: u32 = 1 << 8;
4962#[unstable(feature = "stdarch_s390x", issue = "135681")]
4963pub const __VEC_CLASS_FP_NORMAL: u32 = __VEC_CLASS_FP_NORMAL_P | __VEC_CLASS_FP_NORMAL_N;
4964#[unstable(feature = "stdarch_s390x", issue = "135681")]
4965pub const __VEC_CLASS_FP_SUBNORMAL_P: u32 = 1 << 7;
4966#[unstable(feature = "stdarch_s390x", issue = "135681")]
4967pub const __VEC_CLASS_FP_SUBNORMAL_N: u32 = 1 << 6;
4968#[unstable(feature = "stdarch_s390x", issue = "135681")]
4969pub const __VEC_CLASS_FP_SUBNORMAL: u32 = __VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N;
4970#[unstable(feature = "stdarch_s390x", issue = "135681")]
4971pub const __VEC_CLASS_FP_INFINITY_P: u32 = 1 << 5;
4972#[unstable(feature = "stdarch_s390x", issue = "135681")]
4973pub const __VEC_CLASS_FP_INFINITY_N: u32 = 1 << 4;
4974#[unstable(feature = "stdarch_s390x", issue = "135681")]
4975pub const __VEC_CLASS_FP_INFINITY: u32 = __VEC_CLASS_FP_INFINITY_P | __VEC_CLASS_FP_INFINITY_N;
4976#[unstable(feature = "stdarch_s390x", issue = "135681")]
4977pub const __VEC_CLASS_FP_QNAN_P: u32 = 1 << 3;
4978#[unstable(feature = "stdarch_s390x", issue = "135681")]
4979pub const __VEC_CLASS_FP_QNAN_N: u32 = 1 << 2;
4980#[unstable(feature = "stdarch_s390x", issue = "135681")]
4981pub const __VEC_CLASS_FP_QNAN: u32 = __VEC_CLASS_FP_QNAN_P | __VEC_CLASS_FP_QNAN_N;
4982#[unstable(feature = "stdarch_s390x", issue = "135681")]
4983pub const __VEC_CLASS_FP_SNAN_P: u32 = 1 << 1;
4984#[unstable(feature = "stdarch_s390x", issue = "135681")]
4985pub const __VEC_CLASS_FP_SNAN_N: u32 = 1 << 0;
4986#[unstable(feature = "stdarch_s390x", issue = "135681")]
4987pub const __VEC_CLASS_FP_SNAN: u32 = __VEC_CLASS_FP_SNAN_P | __VEC_CLASS_FP_SNAN_N;
4988#[unstable(feature = "stdarch_s390x", issue = "135681")]
4989pub const __VEC_CLASS_FP_NAN: u32 = __VEC_CLASS_FP_QNAN | __VEC_CLASS_FP_SNAN;
4990#[unstable(feature = "stdarch_s390x", issue = "135681")]
4991pub const __VEC_CLASS_FP_NOT_NORMAL: u32 =
4992    __VEC_CLASS_FP_NAN | __VEC_CLASS_FP_SUBNORMAL | __VEC_CLASS_FP_ZERO | __VEC_CLASS_FP_INFINITY;
4993
4994/// Vector Floating-Point Test Data Class
4995///
4996/// You can use the `__VEC_CLASS_FP_*` constants as the argument for this operand
4997#[inline]
4998#[target_feature(enable = "vector")]
4999#[unstable(feature = "stdarch_s390x", issue = "135681")]
5000pub unsafe fn vec_fp_test_data_class<T: sealed::VectorFpTestDataClass, const CLASS: u32>(
5001    a: T,
5002    c: *mut i32,
5003) -> T::Result {
5004    let (x, y) = a.vec_fp_test_data_class::<CLASS>();
5005    c.write(y);
5006    x
5007}
5008
5009/// All Elements Not a Number
5010#[inline]
5011#[target_feature(enable = "vector")]
5012#[unstable(feature = "stdarch_s390x", issue = "135681")]
5013pub unsafe fn vec_all_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5014    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 0)
5015}
5016
5017/// All Elements Numeric
5018#[inline]
5019#[target_feature(enable = "vector")]
5020#[unstable(feature = "stdarch_s390x", issue = "135681")]
5021pub unsafe fn vec_all_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5022    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 3)
5023}
5024
5025/// Any Elements Not a Number
5026#[inline]
5027#[target_feature(enable = "vector")]
5028#[unstable(feature = "stdarch_s390x", issue = "135681")]
5029pub unsafe fn vec_any_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5030    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 3)
5031}
5032
5033/// Any Elements Numeric
5034#[inline]
5035#[target_feature(enable = "vector")]
5036#[unstable(feature = "stdarch_s390x", issue = "135681")]
5037pub unsafe fn vec_any_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5038    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 0)
5039}
5040
5041/// Vector Test under Mask
5042#[inline]
5043#[target_feature(enable = "vector")]
5044#[unstable(feature = "stdarch_s390x", issue = "135681")]
5045pub unsafe fn vec_test_mask<T: sealed::VectorTestMask>(a: T, b: T::Mask) -> i32 {
5046    // I can't find much information about this, but this might just be a check for whether the
5047    // bitwise and of a and b is non-zero?
5048    a.vec_test_mask(b)
5049}
5050
5051/// Vector Search String
5052#[inline]
5053#[target_feature(enable = "vector")]
5054#[unstable(feature = "stdarch_s390x", issue = "135681")]
5055pub unsafe fn vec_search_string_cc<T: sealed::VectorSearchString>(
5056    a: T,
5057    b: T,
5058    c: vector_unsigned_char,
5059    d: *mut i32,
5060) -> vector_unsigned_char {
5061    let (x, y) = a.vec_search_string_cc(b, c);
5062    unsafe { d.write(y) };
5063    x
5064}
5065
5066/// Vector Search String Until Zero
5067#[inline]
5068#[target_feature(enable = "vector")]
5069#[unstable(feature = "stdarch_s390x", issue = "135681")]
5070pub unsafe fn vec_search_string_until_zero_cc<T: sealed::VectorSearchString>(
5071    a: T,
5072    b: T,
5073    c: vector_unsigned_char,
5074    d: *mut i32,
5075) -> vector_unsigned_char {
5076    let (x, y) = a.vec_search_string_until_zero_cc(b, c);
5077    unsafe { d.write(y) };
5078    x
5079}
5080
5081/// Vector Convert from float (even elements) to double
5082#[inline]
5083#[target_feature(enable = "vector-enhancements-1")]
5084#[unstable(feature = "stdarch_s390x", issue = "135681")]
5085// FIXME: this emits `vflls` where `vldeb` is expected
5086// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vldeb))]
5087pub unsafe fn vec_doublee(a: vector_float) -> vector_double {
5088    let even = simd_shuffle::<_, _, f32x2>(a, a, const { u32x2::from_array([0, 2]) });
5089    simd_as(even)
5090}
5091
5092/// Vector Convert from double to float (even elements)
5093#[inline]
5094#[target_feature(enable = "vector-enhancements-1")]
5095#[unstable(feature = "stdarch_s390x", issue = "135681")]
5096// FIXME: the C version uses a shuffle mask with poison; we can't do that
5097// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vledb))]
5098pub unsafe fn vec_floate(a: vector_double) -> vector_float {
5099    let truncated: f32x2 = simd_as(a);
5100    simd_shuffle(
5101        truncated,
5102        truncated,
5103        const { u32x4::from_array([0, 0, 1, 1]) },
5104    )
5105}
5106
5107/// Vector Convert from int to float
5108#[inline]
5109#[target_feature(enable = "vector")]
5110#[unstable(feature = "stdarch_s390x", issue = "135681")]
5111pub unsafe fn vec_float(a: impl sealed::VectorFloat) -> vector_float {
5112    a.vec_float()
5113}
5114
5115/// Vector Convert from long long to double
5116#[inline]
5117#[target_feature(enable = "vector")]
5118#[unstable(feature = "stdarch_s390x", issue = "135681")]
5119pub unsafe fn vec_double(a: impl sealed::VectorDouble) -> vector_double {
5120    a.vec_double()
5121}
5122
5123/// Vector Sign Extend to Doubleword
5124#[inline]
5125#[target_feature(enable = "vector")]
5126#[unstable(feature = "stdarch_s390x", issue = "135681")]
5127pub unsafe fn vec_extend_s64(a: impl sealed::VectorExtendSigned64) -> vector_signed_long_long {
5128    a.vec_extend_s64()
5129}
5130
5131/// Vector Convert floating point to signed
5132#[inline]
5133#[target_feature(enable = "vector")]
5134#[unstable(feature = "stdarch_s390x", issue = "135681")]
5135pub unsafe fn vec_signed<T: sealed::VectorSigned>(a: T) -> T::Result {
5136    a.vec_signed()
5137}
5138
5139/// Vector Convert floating point to unsigned
5140#[inline]
5141#[target_feature(enable = "vector")]
5142#[unstable(feature = "stdarch_s390x", issue = "135681")]
5143pub unsafe fn vec_unsigned<T: sealed::VectorUnsigned>(a: T) -> T::Result {
5144    a.vec_unsigned()
5145}
5146
5147/// Vector Copy Until Zero
5148#[inline]
5149#[target_feature(enable = "vector")]
5150#[unstable(feature = "stdarch_s390x", issue = "135681")]
5151pub unsafe fn vec_cp_until_zero<T: sealed::VectorCopyUntilZero>(a: T) -> T {
5152    a.vec_cp_until_zero()
5153}
5154
5155/// Vector Copy Until Zero
5156#[inline]
5157#[target_feature(enable = "vector")]
5158#[unstable(feature = "stdarch_s390x", issue = "135681")]
5159pub unsafe fn vec_cp_until_zero_cc<T: sealed::VectorCopyUntilZeroCC>(a: T, cc: *mut i32) -> T {
5160    let (x, y) = a.vec_cp_until_zero_cc();
5161    unsafe { cc.write(y) };
5162    x
5163}
5164
5165/// Vector Multiply Sum Logical
5166#[inline]
5167#[target_feature(enable = "vector-enhancements-1")]
5168#[unstable(feature = "stdarch_s390x", issue = "135681")]
5169#[cfg_attr(
5170    all(test, target_feature = "vector-enhancements-1"),
5171    assert_instr(vmslg, D = 4)
5172)]
5173pub unsafe fn vec_msum_u128<const D: u32>(
5174    a: vector_unsigned_long_long,
5175    b: vector_unsigned_long_long,
5176    c: vector_unsigned_char,
5177) -> vector_unsigned_char {
5178    const {
5179        if !matches!(D, 0 | 4 | 8 | 12) {
5180            panic!("D needs to be one of 0, 4, 8, 12");
5181        }
5182    };
5183    transmute(vmslg(a, b, transmute(c), D))
5184}
5185
5186/// Vector Shift Left Double by Byte
5187#[inline]
5188#[target_feature(enable = "vector")]
5189#[unstable(feature = "stdarch_s390x", issue = "135681")]
5190pub unsafe fn vec_sld<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5191    static_assert_uimm_bits!(C, 4);
5192    a.vec_sld::<C>(b)
5193}
5194
5195/// Vector Shift Left Double by Word
5196#[inline]
5197#[target_feature(enable = "vector")]
5198#[unstable(feature = "stdarch_s390x", issue = "135681")]
5199pub unsafe fn vec_sldw<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5200    static_assert_uimm_bits!(C, 2);
5201    a.vec_sldw::<C>(b)
5202}
5203
5204/// Vector Shift Left Double by Bit
5205#[inline]
5206#[target_feature(enable = "vector-enhancements-2")]
5207#[unstable(feature = "stdarch_s390x", issue = "135681")]
5208pub unsafe fn vec_sldb<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5209    static_assert_uimm_bits!(C, 3);
5210    a.vec_sldb::<C>(b)
5211}
5212
5213/// Vector Shift Right Double by Bit
5214#[inline]
5215#[target_feature(enable = "vector-enhancements-2")]
5216#[unstable(feature = "stdarch_s390x", issue = "135681")]
5217pub unsafe fn vec_srdb<T: sealed::VectorSrdb, const C: u32>(a: T, b: T) -> T {
5218    static_assert_uimm_bits!(C, 3);
5219    a.vec_srdb::<C>(b)
5220}
5221
5222/// Vector Compare Ranges
5223#[inline]
5224#[target_feature(enable = "vector")]
5225#[unstable(feature = "stdarch_s390x", issue = "135681")]
5226pub unsafe fn vec_cmprg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5227    a.vstrc::<{ FindImm::Eq as u32 }>(b, c)
5228}
5229
5230/// Vector Compare Not in Ranges
5231#[inline]
5232#[target_feature(enable = "vector")]
5233#[unstable(feature = "stdarch_s390x", issue = "135681")]
5234pub unsafe fn vec_cmpnrg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5235    a.vstrc::<{ FindImm::Ne as u32 }>(b, c)
5236}
5237
5238/// Vector Compare Ranges Index
5239#[inline]
5240#[target_feature(enable = "vector")]
5241#[unstable(feature = "stdarch_s390x", issue = "135681")]
5242pub unsafe fn vec_cmprg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5243    a.vstrc::<{ FindImm::EqIdx as u32 }>(b, c)
5244}
5245
5246/// Vector Compare Not in Ranges Index
5247#[inline]
5248#[target_feature(enable = "vector")]
5249#[unstable(feature = "stdarch_s390x", issue = "135681")]
5250pub unsafe fn vec_cmpnrg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5251    a.vstrc::<{ FindImm::NeIdx as u32 }>(b, c)
5252}
5253
5254/// Vector Compare Ranges with Condition Code
5255#[inline]
5256#[target_feature(enable = "vector")]
5257#[unstable(feature = "stdarch_s390x", issue = "135681")]
5258pub unsafe fn vec_cmprg_cc<T: sealed::VectorCompareRange>(
5259    a: T,
5260    b: T,
5261    c: T,
5262    d: *mut i32,
5263) -> T::Result {
5264    let (x, y) = a.vstrcs::<{ FindImm::Eq as u32 }>(b, c);
5265    d.write(y);
5266    x
5267}
5268
5269/// Vector Compare Not in Ranges with Condition Code
5270#[inline]
5271#[target_feature(enable = "vector")]
5272#[unstable(feature = "stdarch_s390x", issue = "135681")]
5273pub unsafe fn vec_cmpnrg_cc<T: sealed::VectorCompareRange>(
5274    a: T,
5275    b: T,
5276    c: T,
5277    d: *mut i32,
5278) -> T::Result {
5279    let (x, y) = a.vstrcs::<{ FindImm::Ne as u32 }>(b, c);
5280    d.write(y);
5281    x
5282}
5283
5284/// Vector Compare Ranges Index with Condition Code
5285#[inline]
5286#[target_feature(enable = "vector")]
5287#[unstable(feature = "stdarch_s390x", issue = "135681")]
5288pub unsafe fn vec_cmprg_idx_cc<T: sealed::VectorCompareRange>(
5289    a: T,
5290    b: T,
5291    c: T,
5292    d: *mut i32,
5293) -> T::Result {
5294    let (x, y) = a.vstrcs::<{ FindImm::EqIdx as u32 }>(b, c);
5295    d.write(y);
5296    x
5297}
5298
5299/// Vector Compare Not in Ranges Index with Condition Code
5300#[inline]
5301#[target_feature(enable = "vector")]
5302#[unstable(feature = "stdarch_s390x", issue = "135681")]
5303pub unsafe fn vec_cmpnrg_idx_cc<T: sealed::VectorCompareRange>(
5304    a: T,
5305    b: T,
5306    c: T,
5307    d: *mut i32,
5308) -> T::Result {
5309    let (x, y) = a.vstrcs::<{ FindImm::NeIdx as u32 }>(b, c);
5310    d.write(y);
5311    x
5312}
5313
5314/// Vector Compare Ranges or Zero Index
5315#[inline]
5316#[target_feature(enable = "vector")]
5317#[unstable(feature = "stdarch_s390x", issue = "135681")]
5318pub unsafe fn vec_cmprg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5319    a.vstrcz::<{ FindImm::EqIdx as u32 }>(b, c)
5320}
5321
5322/// Vector Compare Not in Ranges or Zero Index
5323#[inline]
5324#[target_feature(enable = "vector")]
5325#[unstable(feature = "stdarch_s390x", issue = "135681")]
5326pub unsafe fn vec_cmpnrg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5327    a.vstrcz::<{ FindImm::NeIdx as u32 }>(b, c)
5328}
5329
5330/// Vector Compare Ranges or Zero Index with Condition Code
5331#[inline]
5332#[target_feature(enable = "vector")]
5333#[unstable(feature = "stdarch_s390x", issue = "135681")]
5334pub unsafe fn vec_cmprg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5335    a: T,
5336    b: T,
5337    c: T,
5338    d: *mut i32,
5339) -> T::Result {
5340    let (x, y) = a.vstrczs::<{ FindImm::EqIdx as u32 }>(b, c);
5341    d.write(y);
5342    x
5343}
5344
5345/// Vector Compare Not in Ranges or Zero Index with Condition Code
5346#[inline]
5347#[target_feature(enable = "vector")]
5348#[unstable(feature = "stdarch_s390x", issue = "135681")]
5349pub unsafe fn vec_cmpnrg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5350    a: T,
5351    b: T,
5352    c: T,
5353    d: *mut i32,
5354) -> T::Result {
5355    let (x, y) = a.vstrczs::<{ FindImm::NeIdx as u32 }>(b, c);
5356    d.write(y);
5357    x
5358}
5359
5360/// Vector Compare Equal
5361#[inline]
5362#[target_feature(enable = "vector")]
5363#[unstable(feature = "stdarch_s390x", issue = "135681")]
5364pub unsafe fn vec_cmpeq<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5365    a.vec_cmpeq(b)
5366}
5367
5368/// Vector Compare Not Equal
5369#[inline]
5370#[target_feature(enable = "vector")]
5371#[unstable(feature = "stdarch_s390x", issue = "135681")]
5372pub unsafe fn vec_cmpne<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5373    a.vec_cmpne(b)
5374}
5375
5376/// Vector Compare Greater Than
5377#[inline]
5378#[target_feature(enable = "vector")]
5379#[unstable(feature = "stdarch_s390x", issue = "135681")]
5380pub unsafe fn vec_cmpgt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5381    a.vec_cmpgt(b)
5382}
5383
5384/// Vector Compare Greater Than or Equal
5385#[inline]
5386#[target_feature(enable = "vector")]
5387#[unstable(feature = "stdarch_s390x", issue = "135681")]
5388pub unsafe fn vec_cmpge<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5389    a.vec_cmpge(b)
5390}
5391
5392/// Vector Compare Less
5393#[inline]
5394#[target_feature(enable = "vector")]
5395#[unstable(feature = "stdarch_s390x", issue = "135681")]
5396pub unsafe fn vec_cmplt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5397    a.vec_cmplt(b)
5398}
5399
5400/// Vector Compare Less Than or Equal
5401#[inline]
5402#[target_feature(enable = "vector")]
5403#[unstable(feature = "stdarch_s390x", issue = "135681")]
5404pub unsafe fn vec_cmple<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5405    a.vec_cmple(b)
5406}
5407
5408/// Vector Compare Equal Index
5409#[inline]
5410#[target_feature(enable = "vector")]
5411#[unstable(feature = "stdarch_s390x", issue = "135681")]
5412pub unsafe fn vec_cmpeq_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5413    a.vec_cmpeq_idx(b)
5414}
5415/// Vector Compare Not Equal Index
5416#[inline]
5417#[target_feature(enable = "vector")]
5418#[unstable(feature = "stdarch_s390x", issue = "135681")]
5419pub unsafe fn vec_cmpne_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5420    a.vec_cmpne_idx(b)
5421}
5422/// Vector Compare Equal Index with Condition Code
5423#[inline]
5424#[target_feature(enable = "vector")]
5425#[unstable(feature = "stdarch_s390x", issue = "135681")]
5426pub unsafe fn vec_cmpeq_idx_cc<T: sealed::VectorEqualityIdx>(
5427    a: T,
5428    b: T,
5429    cc: *mut i32,
5430) -> T::Result {
5431    let (x, y) = a.vec_cmpeq_idx_cc(b);
5432    unsafe { cc.write(y) };
5433    x
5434}
5435/// Vector Compare Not Equal Index with Condition Code
5436#[inline]
5437#[target_feature(enable = "vector")]
5438#[unstable(feature = "stdarch_s390x", issue = "135681")]
5439pub unsafe fn vec_cmpne_idx_cc<T: sealed::VectorEqualityIdx>(
5440    a: T,
5441    b: T,
5442    cc: *mut i32,
5443) -> T::Result {
5444    let (x, y) = a.vec_cmpne_idx_cc(b);
5445    unsafe { cc.write(y) };
5446    x
5447}
5448/// Vector Compare Equal or Zero Index
5449#[inline]
5450#[target_feature(enable = "vector")]
5451#[unstable(feature = "stdarch_s390x", issue = "135681")]
5452pub unsafe fn vec_cmpeq_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5453    a.vec_cmpeq_or_0_idx(b)
5454}
5455/// Vector Compare Not Equal or Zero Index
5456#[inline]
5457#[target_feature(enable = "vector")]
5458#[unstable(feature = "stdarch_s390x", issue = "135681")]
5459pub unsafe fn vec_cmpne_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5460    a.vec_cmpne_or_0_idx(b)
5461}
5462/// Vector Compare Equal or Zero Index with Condition Code
5463#[inline]
5464#[target_feature(enable = "vector")]
5465#[unstable(feature = "stdarch_s390x", issue = "135681")]
5466pub unsafe fn vec_cmpeq_or_0_idx_cc<T: sealed::VectorEqualityIdx>(
5467    a: T,
5468    b: T,
5469    cc: *mut i32,
5470) -> T::Result {
5471    let (x, y) = a.vec_cmpeq_or_0_idx_cc(b);
5472    unsafe { cc.write(y) };
5473    x
5474}
5475/// Vector Compare Not Equal or Zero Index with Condition Code
5476#[inline]
5477#[target_feature(enable = "vector")]
5478#[unstable(feature = "stdarch_s390x", issue = "135681")]
5479pub unsafe fn vec_cmpne_or_0_idx_cc<T: sealed::VectorEqualityIdx>(
5480    a: T,
5481    b: T,
5482    cc: *mut i32,
5483) -> T::Result {
5484    let (x, y) = a.vec_cmpne_or_0_idx_cc(b);
5485    unsafe { cc.write(y) };
5486    x
5487}
5488
5489/// All Elements Equal
5490#[inline]
5491#[target_feature(enable = "vector")]
5492#[unstable(feature = "stdarch_s390x", issue = "135681")]
5493pub unsafe fn vec_all_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5494    simd_reduce_all(vec_cmpeq(a, b)) as i32 as i32
5495}
5496
5497/// All Elements Not Equal
5498#[inline]
5499#[target_feature(enable = "vector")]
5500#[unstable(feature = "stdarch_s390x", issue = "135681")]
5501pub unsafe fn vec_all_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5502    simd_reduce_all(vec_cmpne(a, b)) as i32
5503}
5504
5505/// Any Element Equal
5506#[inline]
5507#[target_feature(enable = "vector")]
5508#[unstable(feature = "stdarch_s390x", issue = "135681")]
5509pub unsafe fn vec_any_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5510    simd_reduce_any(vec_cmpeq(a, b)) as i32
5511}
5512
5513/// Any Element Not Equal
5514#[inline]
5515#[target_feature(enable = "vector")]
5516#[unstable(feature = "stdarch_s390x", issue = "135681")]
5517pub unsafe fn vec_any_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5518    simd_reduce_any(vec_cmpne(a, b)) as i32
5519}
5520
5521/// All Elements Less Than
5522#[inline]
5523#[target_feature(enable = "vector")]
5524#[unstable(feature = "stdarch_s390x", issue = "135681")]
5525pub unsafe fn vec_all_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5526    a.vec_all_lt(b)
5527}
5528
5529/// All Elements Less Than or Equal
5530#[inline]
5531#[target_feature(enable = "vector")]
5532#[unstable(feature = "stdarch_s390x", issue = "135681")]
5533pub unsafe fn vec_all_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5534    a.vec_all_le(b)
5535}
5536
5537/// All Elements Greater Than
5538#[inline]
5539#[target_feature(enable = "vector")]
5540#[unstable(feature = "stdarch_s390x", issue = "135681")]
5541pub unsafe fn vec_all_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5542    a.vec_all_gt(b)
5543}
5544
5545/// All Elements Greater Than or Equal
5546#[inline]
5547#[target_feature(enable = "vector")]
5548#[unstable(feature = "stdarch_s390x", issue = "135681")]
5549pub unsafe fn vec_all_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5550    a.vec_all_ge(b)
5551}
5552
5553/// All Elements Not Less Than
5554#[inline]
5555#[target_feature(enable = "vector")]
5556#[unstable(feature = "stdarch_s390x", issue = "135681")]
5557pub unsafe fn vec_all_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5558    vec_all_ge(a, b)
5559}
5560
5561/// All Elements Not Less Than or Equal
5562#[inline]
5563#[target_feature(enable = "vector")]
5564#[unstable(feature = "stdarch_s390x", issue = "135681")]
5565pub unsafe fn vec_all_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5566    vec_all_gt(a, b)
5567}
5568
5569/// All Elements Not Greater Than
5570#[inline]
5571#[target_feature(enable = "vector")]
5572#[unstable(feature = "stdarch_s390x", issue = "135681")]
5573pub unsafe fn vec_all_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5574    vec_all_le(a, b)
5575}
5576
5577/// All Elements Not Greater Than or Equal
5578#[inline]
5579#[target_feature(enable = "vector")]
5580#[unstable(feature = "stdarch_s390x", issue = "135681")]
5581pub unsafe fn vec_all_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5582    vec_all_lt(a, b)
5583}
5584
5585/// Any Elements Less Than
5586#[inline]
5587#[target_feature(enable = "vector")]
5588#[unstable(feature = "stdarch_s390x", issue = "135681")]
5589pub unsafe fn vec_any_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5590    !vec_all_ge(a, b)
5591}
5592
5593/// Any Elements Less Than or Equal
5594#[inline]
5595#[target_feature(enable = "vector")]
5596#[unstable(feature = "stdarch_s390x", issue = "135681")]
5597pub unsafe fn vec_any_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5598    !vec_all_gt(a, b)
5599}
5600
5601/// Any Elements Greater Than
5602#[inline]
5603#[target_feature(enable = "vector")]
5604#[unstable(feature = "stdarch_s390x", issue = "135681")]
5605pub unsafe fn vec_any_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5606    !vec_all_le(a, b)
5607}
5608
5609/// Any Elements Greater Than or Equal
5610#[inline]
5611#[target_feature(enable = "vector")]
5612#[unstable(feature = "stdarch_s390x", issue = "135681")]
5613pub unsafe fn vec_any_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5614    !vec_all_lt(a, b)
5615}
5616
5617/// Any Elements Not Less Than
5618#[inline]
5619#[target_feature(enable = "vector")]
5620#[unstable(feature = "stdarch_s390x", issue = "135681")]
5621pub unsafe fn vec_any_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5622    vec_any_ge(a, b)
5623}
5624
5625/// Any Elements Not Less Than or Equal
5626#[inline]
5627#[target_feature(enable = "vector")]
5628#[unstable(feature = "stdarch_s390x", issue = "135681")]
5629pub unsafe fn vec_any_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5630    vec_any_gt(a, b)
5631}
5632
5633/// Any Elements Not Greater Than
5634#[inline]
5635#[target_feature(enable = "vector")]
5636#[unstable(feature = "stdarch_s390x", issue = "135681")]
5637pub unsafe fn vec_any_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5638    vec_any_le(a, b)
5639}
5640
5641/// Any Elements Not Greater Than or Equal
5642#[inline]
5643#[target_feature(enable = "vector")]
5644#[unstable(feature = "stdarch_s390x", issue = "135681")]
5645pub unsafe fn vec_any_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5646    vec_any_lt(a, b)
5647}
5648
5649#[cfg(test)]
5650mod tests {
5651    use super::*;
5652
5653    use std::mem::transmute;
5654
5655    use crate::core_arch::simd::*;
5656    use stdarch_test::simd_test;
5657
5658    #[test]
5659    fn reverse_mask() {
5660        assert_eq!(ShuffleMask::<4>::reverse().0, [3, 2, 1, 0]);
5661    }
5662
5663    #[test]
5664    fn mergel_mask() {
5665        assert_eq!(ShuffleMask::<4>::merge_low().0, [2, 6, 3, 7]);
5666    }
5667
5668    #[test]
5669    fn mergeh_mask() {
5670        assert_eq!(ShuffleMask::<4>::merge_high().0, [0, 4, 1, 5]);
5671    }
5672
5673    #[test]
5674    fn pack_mask() {
5675        assert_eq!(ShuffleMask::<4>::pack().0, [1, 3, 5, 7]);
5676    }
5677
5678    #[test]
5679    fn test_vec_mask() {
5680        assert_eq!(
5681            genmask::<0x00FF>(),
5682            [
5683                0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
5684            ]
5685        );
5686    }
5687
5688    #[test]
5689    fn test_genmasks() {
5690        assert_eq!(genmasks(u8::BITS, 3, 5), 28);
5691        assert_eq!(genmasks(u8::BITS, 3, 7), 31);
5692
5693        // If a or b is greater than 8, the operation is performed as if the value gets modulo by 8.
5694        assert_eq!(genmasks(u8::BITS, 3 + 8, 7 + 8), 31);
5695        // If a is greater than b, the operation is perform as if b equals 7.
5696        assert_eq!(genmasks(u8::BITS, 5, 4), genmasks(u8::BITS, 5, 7));
5697
5698        assert_eq!(
5699            genmasks(u16::BITS, 4, 12) as u16,
5700            u16::from_be_bytes([15, -8i8 as u8])
5701        );
5702        assert_eq!(
5703            genmasks(u32::BITS, 4, 29) as u32,
5704            u32::from_be_bytes([15, 0xFF, 0xFF, -4i8 as u8])
5705        );
5706    }
5707
5708    macro_rules! test_vec_1 {
5709        { $name: ident, $fn:ident, f32x4, [$($a:expr),+], ~[$($d:expr),+] } => {
5710            #[simd_test(enable = "vector")]
5711            unsafe fn $name() {
5712                let a: vector_float = transmute(f32x4::new($($a),+));
5713
5714                let d: vector_float = transmute(f32x4::new($($d),+));
5715                let r = transmute(vec_cmple(vec_abs(vec_sub($fn(a), d)), vec_splats(f32::EPSILON)));
5716                let e = m32x4::new(true, true, true, true);
5717                assert_eq!(e, r);
5718            }
5719        };
5720        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($d:expr),+] } => {
5721            test_vec_1! { $name, $fn, $ty -> $ty, [$($a),+], [$($d),+] }
5722        };
5723        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($d:expr),+] } => {
5724            #[simd_test(enable = "vector")]
5725            unsafe fn $name() {
5726                let a: s_t_l!($ty) = transmute($ty::new($($a),+));
5727
5728                let d = $ty_out::new($($d),+);
5729                let r : $ty_out = transmute($fn(a));
5730                assert_eq!(d, r);
5731            }
5732        }
5733    }
5734
5735    macro_rules! test_vec_2 {
5736        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5737            test_vec_2! { $name, $fn, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
5738        };
5739        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5740            test_vec_2! { $name, $fn, $ty, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
5741         };
5742        { $name: ident, $fn:ident, $ty1: ident, $ty2: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5743            #[simd_test(enable = "vector")]
5744            unsafe fn $name() {
5745                let a: s_t_l!($ty1) = transmute($ty1::new($($a),+));
5746                let b: s_t_l!($ty2) = transmute($ty2::new($($b),+));
5747
5748                let d = $ty_out::new($($d),+);
5749                let r : $ty_out = transmute($fn(a, b));
5750                assert_eq!(d, r);
5751            }
5752         };
5753         { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], $d:expr } => {
5754            #[simd_test(enable = "vector")]
5755            unsafe fn $name() {
5756                let a: s_t_l!($ty) = transmute($ty::new($($a),+));
5757                let b: s_t_l!($ty) = transmute($ty::new($($b),+));
5758
5759                let r : $ty_out = transmute($fn(a, b));
5760                assert_eq!($d, r);
5761            }
5762         }
5763   }
5764
5765    #[simd_test(enable = "vector")]
5766    unsafe fn vec_add_i32x4_i32x4() {
5767        let x = i32x4::new(1, 2, 3, 4);
5768        let y = i32x4::new(4, 3, 2, 1);
5769        let x: vector_signed_int = transmute(x);
5770        let y: vector_signed_int = transmute(y);
5771        let z = vec_add(x, y);
5772        assert_eq!(i32x4::splat(5), transmute(z));
5773    }
5774
5775    macro_rules! test_vec_sub {
5776        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5777            test_vec_2! {$name, vec_sub, $ty, [$($a),+], [$($b),+], [$($d),+] }
5778        }
5779    }
5780
5781    test_vec_sub! { test_vec_sub_f32x4, f32x4,
5782    [-1.0, 0.0, 1.0, 2.0],
5783    [2.0, 1.0, -1.0, -2.0],
5784    [-3.0, -1.0, 2.0, 4.0] }
5785
5786    test_vec_sub! { test_vec_sub_f64x2, f64x2,
5787    [-1.0, 0.0],
5788    [2.0, 1.0],
5789    [-3.0, -1.0] }
5790
5791    test_vec_sub! { test_vec_sub_i64x2, i64x2,
5792    [-1, 0],
5793    [2, 1],
5794    [-3, -1] }
5795
5796    test_vec_sub! { test_vec_sub_u64x2, u64x2,
5797    [0, 1],
5798    [1, 0],
5799    [u64::MAX, 1] }
5800
5801    test_vec_sub! { test_vec_sub_i32x4, i32x4,
5802    [-1, 0, 1, 2],
5803    [2, 1, -1, -2],
5804    [-3, -1, 2, 4] }
5805
5806    test_vec_sub! { test_vec_sub_u32x4, u32x4,
5807    [0, 0, 1, 2],
5808    [2, 1, 0, 0],
5809    [4294967294, 4294967295, 1, 2] }
5810
5811    test_vec_sub! { test_vec_sub_i16x8, i16x8,
5812    [-1, 0, 1, 2, -1, 0, 1, 2],
5813    [2, 1, -1, -2, 2, 1, -1, -2],
5814    [-3, -1, 2, 4, -3, -1, 2, 4] }
5815
5816    test_vec_sub! { test_vec_sub_u16x8, u16x8,
5817    [0, 0, 1, 2, 0, 0, 1, 2],
5818    [2, 1, 0, 0, 2, 1, 0, 0],
5819    [65534, 65535, 1, 2, 65534, 65535, 1, 2] }
5820
5821    test_vec_sub! { test_vec_sub_i8x16, i8x16,
5822    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
5823    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
5824    [-3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4] }
5825
5826    test_vec_sub! { test_vec_sub_u8x16, u8x16,
5827    [0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2],
5828    [2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0],
5829    [254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2] }
5830
5831    macro_rules! test_vec_mul {
5832        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5833            test_vec_2! {$name, vec_mul, $ty, [$($a),+], [$($b),+], [$($d),+] }
5834        }
5835    }
5836
5837    test_vec_mul! { test_vec_mul_f32x4, f32x4,
5838    [-1.0, 0.0, 1.0, 2.0],
5839    [2.0, 1.0, -1.0, -2.0],
5840    [-2.0, 0.0, -1.0, -4.0] }
5841
5842    test_vec_mul! { test_vec_mul_f64x2, f64x2,
5843    [-1.0, 0.0],
5844    [2.0, 1.0],
5845    [-2.0, 0.0] }
5846
5847    test_vec_mul! { test_vec_mul_i64x2, i64x2,
5848    [i64::MAX, -4],
5849    [2, 3],
5850    [i64::MAX.wrapping_mul(2), -12] }
5851
5852    test_vec_mul! { test_vec_mul_u64x2, u64x2,
5853    [u64::MAX, 4],
5854    [2, 3],
5855    [u64::MAX.wrapping_mul(2), 12] }
5856
5857    test_vec_mul! { test_vec_mul_i32x4, i32x4,
5858    [-1, 0, 1, 2],
5859    [2, 1, -1, -2],
5860    [-2, 0, -1, -4] }
5861
5862    test_vec_mul! { test_vec_mul_u32x4, u32x4,
5863    [0, u32::MAX - 1, 1, 2],
5864    [5, 6, 7, 8],
5865    [0, 4294967284, 7, 16] }
5866
5867    test_vec_mul! { test_vec_mul_i16x8, i16x8,
5868    [-1, 0, 1, 2, -1, 0, 1, 2],
5869    [2, 1, -1, -2, 2, 1, -1, -2],
5870    [-2, 0, -1, -4, -2, 0, -1, -4] }
5871
5872    test_vec_mul! { test_vec_mul_u16x8, u16x8,
5873    [0, u16::MAX - 1, 1, 2, 3, 4, 5, 6],
5874    [5, 6, 7, 8, 9, 8, 7, 6],
5875    [0, 65524, 7, 16, 27, 32, 35, 36] }
5876
5877    test_vec_mul! { test_vec_mul_i8x16, i8x16,
5878    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
5879    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
5880    [-2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4] }
5881
5882    test_vec_mul! { test_vec_mul_u8x16, u8x16,
5883    [0, u8::MAX - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4],
5884    [5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 0, u8::MAX, 1, 2, 3, 4],
5885    [0, 244, 7, 16, 27, 32, 35, 36, 35, 32, 0, 248, 7, 12, 15, 16] }
5886
5887    macro_rules! test_vec_abs {
5888        { $name: ident, $ty: ident, $a: expr, $d: expr } => {
5889            #[simd_test(enable = "vector")]
5890            unsafe fn $name() {
5891                let a: s_t_l!($ty) = vec_splats($a);
5892                let a: s_t_l!($ty) = vec_abs(a);
5893                let d = $ty::splat($d);
5894                assert_eq!(d, transmute(a));
5895            }
5896        }
5897    }
5898
5899    test_vec_abs! { test_vec_abs_i8, i8x16, -42i8, 42i8 }
5900    test_vec_abs! { test_vec_abs_i16, i16x8, -42i16, 42i16 }
5901    test_vec_abs! { test_vec_abs_i32, i32x4, -42i32, 42i32 }
5902    test_vec_abs! { test_vec_abs_i64, i64x2, -42i64, 42i64 }
5903    test_vec_abs! { test_vec_abs_f32, f32x4, -42f32, 42f32 }
5904    test_vec_abs! { test_vec_abs_f64, f64x2, -42f64, 42f64 }
5905
5906    test_vec_1! { test_vec_nabs, vec_nabs, f32x4,
5907    [core::f32::consts::PI, 1.0, 0.0, -1.0],
5908    [-core::f32::consts::PI, -1.0, 0.0, -1.0] }
5909
5910    test_vec_2! { test_vec_andc, vec_andc, i32x4,
5911    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
5912    [0b00110011, 0b11110011, 0b00001100, 0b10000000],
5913    [0b11001100, 0b00001100, 0b11000000, 0b01001100] }
5914
5915    test_vec_2! { test_vec_and, vec_and, i32x4,
5916    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
5917    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
5918    [0b00000000, 0b11000000, 0b00001100, 0b00000000] }
5919
5920    test_vec_2! { test_vec_nand, vec_nand, i32x4,
5921    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
5922    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
5923    [!0b00000000, !0b11000000, !0b00001100, !0b00000000] }
5924
5925    test_vec_2! { test_vec_orc, vec_orc, u32x4,
5926    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
5927    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
5928    [0b11001100 | !0b00110011, 0b11001100 | !0b11110011, 0b11001100 | !0b00001100, 0b11001100 | !0b00000000] }
5929
5930    test_vec_2! { test_vec_or, vec_or, i32x4,
5931    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
5932    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
5933    [0b11111111, 0b11111111, 0b11001100, 0b11001100] }
5934
5935    test_vec_2! { test_vec_nor, vec_nor, i32x4,
5936    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
5937    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
5938    [!0b11111111, !0b11111111, !0b11001100, !0b11001100] }
5939
5940    test_vec_2! { test_vec_xor, vec_xor, i32x4,
5941    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
5942    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
5943    [0b11111111, 0b00111111, 0b11000000, 0b11001100] }
5944
5945    test_vec_2! { test_vec_eqv, vec_eqv, i32x4,
5946    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
5947    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
5948    [!0b11111111, !0b00111111, !0b11000000, !0b11001100] }
5949
5950    test_vec_1! { test_vec_floor_f32, vec_floor, f32x4,
5951        [1.1, 1.9, -0.5, -0.9],
5952        [1.0, 1.0, -1.0, -1.0]
5953    }
5954
5955    test_vec_1! { test_vec_floor_f64_1, vec_floor, f64x2,
5956        [1.1, 1.9],
5957        [1.0, 1.0]
5958    }
5959    test_vec_1! { test_vec_floor_f64_2, vec_floor, f64x2,
5960        [-0.5, -0.9],
5961        [-1.0, -1.0]
5962    }
5963
5964    test_vec_1! { test_vec_ceil_f32, vec_ceil, f32x4,
5965        [0.1, 0.5, 0.6, 0.9],
5966        [1.0, 1.0, 1.0, 1.0]
5967    }
5968    test_vec_1! { test_vec_ceil_f64_1, vec_ceil, f64x2,
5969        [0.1, 0.5],
5970        [1.0, 1.0]
5971    }
5972    test_vec_1! { test_vec_ceil_f64_2, vec_ceil, f64x2,
5973        [0.6, 0.9],
5974        [1.0, 1.0]
5975    }
5976
5977    test_vec_1! { test_vec_round_f32, vec_round, f32x4,
5978        [0.1, 0.5, 0.6, 0.9],
5979        [0.0, 0.0, 1.0, 1.0]
5980    }
5981
5982    test_vec_1! { test_vec_round_f32_even_odd, vec_round, f32x4,
5983        [0.5, 1.5, 2.5, 3.5],
5984        [0.0, 2.0, 2.0, 4.0]
5985    }
5986
5987    test_vec_1! { test_vec_round_f64_1, vec_round, f64x2,
5988        [0.1, 0.5],
5989        [0.0, 0.0]
5990    }
5991    test_vec_1! { test_vec_round_f64_2, vec_round, f64x2,
5992        [0.6, 0.9],
5993        [1.0, 1.0]
5994    }
5995
5996    test_vec_1! { test_vec_roundc_f32, vec_roundc, f32x4,
5997        [0.1, 0.5, 0.6, 0.9],
5998        [0.0, 0.0, 1.0, 1.0]
5999    }
6000
6001    test_vec_1! { test_vec_roundc_f32_even_odd, vec_roundc, f32x4,
6002        [0.5, 1.5, 2.5, 3.5],
6003        [0.0, 2.0, 2.0, 4.0]
6004    }
6005
6006    test_vec_1! { test_vec_roundc_f64_1, vec_roundc, f64x2,
6007        [0.1, 0.5],
6008        [0.0, 0.0]
6009    }
6010    test_vec_1! { test_vec_roundc_f64_2, vec_roundc, f64x2,
6011        [0.6, 0.9],
6012        [1.0, 1.0]
6013    }
6014
6015    test_vec_1! { test_vec_rint_f32, vec_rint, f32x4,
6016        [0.1, 0.5, 0.6, 0.9],
6017        [0.0, 0.0, 1.0, 1.0]
6018    }
6019
6020    test_vec_1! { test_vec_rint_f32_even_odd, vec_rint, f32x4,
6021        [0.5, 1.5, 2.5, 3.5],
6022        [0.0, 2.0, 2.0, 4.0]
6023    }
6024
6025    test_vec_1! { test_vec_rint_f64_1, vec_rint, f64x2,
6026        [0.1, 0.5],
6027        [0.0, 0.0]
6028    }
6029    test_vec_1! { test_vec_rint_f64_2, vec_rint, f64x2,
6030        [0.6, 0.9],
6031        [1.0, 1.0]
6032    }
6033
6034    test_vec_2! { test_vec_sll, vec_sll, i32x4, u8x16 -> i32x4,
6035    [1, 1, 1, 1],
6036    [0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8],
6037    [1 << 2, 1 << 3, 1 << 4, 1] }
6038
6039    test_vec_2! { test_vec_srl, vec_srl, i32x4, u8x16 -> i32x4,
6040    [0b1000, 0b1000, 0b1000, 0b1000],
6041    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6042    [4, 2, 1, 8] }
6043
6044    test_vec_2! { test_vec_sral_pos, vec_sral, u32x4, u8x16 -> i32x4,
6045    [0b1000, 0b1000, 0b1000, 0b1000],
6046    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6047    [4, 2, 1, 8] }
6048
6049    test_vec_2! { test_vec_sral_neg, vec_sral, i32x4, u8x16 -> i32x4,
6050    [-8, -8, -8, -8],
6051    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6052    [-4, -2, -1, -8] }
6053
6054    test_vec_1! { test_vec_reve_f32, vec_reve, f32x4,
6055        [0.1, 0.5, 0.6, 0.9],
6056        [0.9, 0.6, 0.5, 0.1]
6057    }
6058
6059    test_vec_1! { test_vec_revb_u32, vec_revb, u32x4,
6060        [0xAABBCCDD, 0xEEFF0011, 0x22334455, 0x66778899],
6061        [0xDDCCBBAA, 0x1100FFEE, 0x55443322, 0x99887766]
6062    }
6063
6064    test_vec_2! { test_vec_mergeh_u32, vec_mergeh, u32x4,
6065        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6066        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6067        [0xAAAAAAAA, 0x00000000, 0xBBBBBBBB, 0x11111111]
6068    }
6069
6070    test_vec_2! { test_vec_mergel_u32, vec_mergel, u32x4,
6071        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6072        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6073        [0xCCCCCCCC, 0x22222222, 0xDDDDDDDD, 0x33333333]
6074    }
6075
6076    macro_rules! test_vec_perm {
6077        {$name:ident,
6078         $shorttype:ident, $longtype:ident,
6079         [$($a:expr),+], [$($b:expr),+], [$($c:expr),+], [$($d:expr),+]} => {
6080            #[simd_test(enable = "vector")]
6081            unsafe fn $name() {
6082                let a: $longtype = transmute($shorttype::new($($a),+));
6083                let b: $longtype = transmute($shorttype::new($($b),+));
6084                let c: vector_unsigned_char = transmute(u8x16::new($($c),+));
6085                let d = $shorttype::new($($d),+);
6086
6087                let r: $shorttype = transmute(vec_perm(a, b, c));
6088                assert_eq!(d, r);
6089            }
6090        }
6091    }
6092
6093    test_vec_perm! {test_vec_perm_u8x16,
6094    u8x16, vector_unsigned_char,
6095    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6096    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6097    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6098     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6099    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6100    test_vec_perm! {test_vec_perm_i8x16,
6101    i8x16, vector_signed_char,
6102    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6103    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6104    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6105     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6106    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6107
6108    test_vec_perm! {test_vec_perm_m8x16,
6109    m8x16, vector_bool_char,
6110    [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false],
6111    [true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true],
6112    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6113     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6114    [false, false, true, true, false, false, true, true, false, false, true, true, false, false, true, true]}
6115    test_vec_perm! {test_vec_perm_u16x8,
6116    u16x8, vector_unsigned_short,
6117    [0, 1, 2, 3, 4, 5, 6, 7],
6118    [10, 11, 12, 13, 14, 15, 16, 17],
6119    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6120     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6121    [0, 10, 1, 11, 2, 12, 3, 13]}
6122    test_vec_perm! {test_vec_perm_i16x8,
6123    i16x8, vector_signed_short,
6124    [0, 1, 2, 3, 4, 5, 6, 7],
6125    [10, 11, 12, 13, 14, 15, 16, 17],
6126    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6127     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6128    [0, 10, 1, 11, 2, 12, 3, 13]}
6129    test_vec_perm! {test_vec_perm_m16x8,
6130    m16x8, vector_bool_short,
6131    [false, false, false, false, false, false, false, false],
6132    [true, true, true, true, true, true, true, true],
6133    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6134     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6135    [false, true, false, true, false, true, false, true]}
6136
6137    test_vec_perm! {test_vec_perm_u32x4,
6138    u32x4, vector_unsigned_int,
6139    [0, 1, 2, 3],
6140    [10, 11, 12, 13],
6141    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6142     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6143    [0, 10, 1, 11]}
6144    test_vec_perm! {test_vec_perm_i32x4,
6145    i32x4, vector_signed_int,
6146    [0, 1, 2, 3],
6147    [10, 11, 12, 13],
6148    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6149     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6150    [0, 10, 1, 11]}
6151    test_vec_perm! {test_vec_perm_m32x4,
6152    m32x4, vector_bool_int,
6153    [false, false, false, false],
6154    [true, true, true, true],
6155    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6156     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6157    [false, true, false, true]}
6158    test_vec_perm! {test_vec_perm_f32x4,
6159    f32x4, vector_float,
6160    [0.0, 1.0, 2.0, 3.0],
6161    [1.0, 1.1, 1.2, 1.3],
6162    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6163     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6164    [0.0, 1.0, 1.0, 1.1]}
6165
6166    test_vec_1! { test_vec_sqrt, vec_sqrt, f32x4,
6167    [core::f32::consts::PI, 1.0, 25.0, 2.0],
6168    [core::f32::consts::PI.sqrt(), 1.0, 5.0, core::f32::consts::SQRT_2] }
6169
6170    test_vec_2! { test_vec_find_any_eq, vec_find_any_eq, i32x4, i32x4 -> u32x4,
6171        [1, -2, 3, -4],
6172        [-5, 3, -7, 8],
6173        [0, 0, 0xFFFFFFFF, 0]
6174    }
6175
6176    test_vec_2! { test_vec_find_any_ne, vec_find_any_ne, i32x4, i32x4 -> u32x4,
6177        [1, -2, 3, -4],
6178        [-5, 3, -7, 8],
6179        [0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF]
6180    }
6181
6182    test_vec_2! { test_vec_find_any_eq_idx_1, vec_find_any_eq_idx, i32x4, i32x4 -> u32x4,
6183        [1, 2, 3, 4],
6184        [5, 3, 7, 8],
6185        [0, 8, 0, 0]
6186    }
6187    test_vec_2! { test_vec_find_any_eq_idx_2, vec_find_any_eq_idx, i32x4, i32x4 -> u32x4,
6188        [1, 2, 3, 4],
6189        [5, 6, 7, 8],
6190        [0, 16, 0, 0]
6191    }
6192
6193    test_vec_2! { test_vec_find_any_ne_idx_1, vec_find_any_ne_idx, i32x4, i32x4 -> u32x4,
6194        [1, 2, 3, 4],
6195        [1, 5, 3, 4],
6196        [0, 4, 0, 0]
6197    }
6198    test_vec_2! { test_vec_find_any_ne_idx_2, vec_find_any_ne_idx, i32x4, i32x4 -> u32x4,
6199        [1, 2, 3, 4],
6200        [1, 2, 3, 4],
6201        [0, 16, 0, 0]
6202    }
6203
6204    test_vec_2! { test_vec_find_any_eq_or_0_idx_1, vec_find_any_eq_or_0_idx, i32x4, i32x4 -> u32x4,
6205        [1, 2, 0, 4],
6206        [5, 6, 7, 8],
6207        [0, 8, 0, 0]
6208    }
6209    test_vec_2! { test_vec_find_any_ne_or_0_idx_1, vec_find_any_ne_or_0_idx, i32x4, i32x4 -> u32x4,
6210        [1, 2, 0, 4],
6211        [1, 2, 3, 4],
6212        [0, 8, 0, 0]
6213    }
6214
6215    #[simd_test(enable = "vector")]
6216    fn test_vec_find_any_eq_cc() {
6217        let mut c = 0i32;
6218
6219        let a = vector_unsigned_int([1, 2, 3, 4]);
6220        let b = vector_unsigned_int([5, 3, 7, 8]);
6221
6222        let d = unsafe { vec_find_any_eq_cc(a, b, &mut c) };
6223        assert_eq!(c, 1);
6224        assert_eq!(d.as_array(), &[0, 0, -1, 0]);
6225
6226        let a = vector_unsigned_int([1, 2, 3, 4]);
6227        let b = vector_unsigned_int([5, 6, 7, 8]);
6228        let d = unsafe { vec_find_any_eq_cc(a, b, &mut c) };
6229        assert_eq!(c, 3);
6230        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6231    }
6232
6233    #[simd_test(enable = "vector")]
6234    fn test_vec_find_any_ne_cc() {
6235        let mut c = 0i32;
6236
6237        let a = vector_unsigned_int([1, 2, 3, 4]);
6238        let b = vector_unsigned_int([5, 3, 7, 8]);
6239
6240        let d = unsafe { vec_find_any_ne_cc(a, b, &mut c) };
6241        assert_eq!(c, 1);
6242        assert_eq!(d.as_array(), &[-1, -1, 0, -1]);
6243
6244        let a = vector_unsigned_int([1, 2, 3, 4]);
6245        let b = vector_unsigned_int([1, 2, 3, 4]);
6246        let d = unsafe { vec_find_any_ne_cc(a, b, &mut c) };
6247        assert_eq!(c, 3);
6248        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6249    }
6250
6251    #[simd_test(enable = "vector")]
6252    fn test_vec_find_any_eq_idx_cc() {
6253        let mut c = 0i32;
6254
6255        let a = vector_unsigned_int([1, 2, 3, 4]);
6256        let b = vector_unsigned_int([5, 3, 7, 8]);
6257
6258        let d = unsafe { vec_find_any_eq_idx_cc(a, b, &mut c) };
6259        assert_eq!(c, 1);
6260        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
6261
6262        let a = vector_unsigned_int([1, 2, 3, 4]);
6263        let b = vector_unsigned_int([5, 6, 7, 8]);
6264        let d = unsafe { vec_find_any_eq_idx_cc(a, b, &mut c) };
6265        assert_eq!(c, 3);
6266        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6267    }
6268
6269    #[simd_test(enable = "vector")]
6270    fn test_vec_find_any_ne_idx_cc() {
6271        let mut c = 0i32;
6272
6273        let a = vector_unsigned_int([5, 2, 3, 4]);
6274        let b = vector_unsigned_int([5, 3, 7, 8]);
6275
6276        let d = unsafe { vec_find_any_ne_idx_cc(a, b, &mut c) };
6277        assert_eq!(c, 1);
6278        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6279
6280        let a = vector_unsigned_int([1, 2, 3, 4]);
6281        let b = vector_unsigned_int([1, 2, 3, 4]);
6282        let d = unsafe { vec_find_any_ne_idx_cc(a, b, &mut c) };
6283        assert_eq!(c, 3);
6284        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6285    }
6286
6287    #[simd_test(enable = "vector")]
6288    fn test_vec_find_any_eq_or_0_idx_cc() {
6289        let mut c = 0i32;
6290
6291        // if no element of a matches any element of b with an equal value, and there is at least one element from a with a value of 0
6292        let a = vector_unsigned_int([0, 1, 2, 3]);
6293        let b = vector_unsigned_int([4, 5, 6, 7]);
6294        let d = unsafe { vec_find_any_eq_or_0_idx_cc(a, b, &mut c) };
6295        assert_eq!(c, 0);
6296        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6297
6298        // if at least one element of a matches any element of b with an equal value, and no elements of a with a value of 0
6299        let a = vector_unsigned_int([1, 2, 3, 4]);
6300        let b = vector_unsigned_int([5, 2, 3, 4]);
6301        let d = unsafe { vec_find_any_eq_or_0_idx_cc(a, b, &mut c) };
6302        assert_eq!(c, 1);
6303        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6304
6305        // if at least one element of a matches any element of b with an equal value, and there is at least one element from a has a value of 0
6306        let a = vector_unsigned_int([1, 2, 3, 0]);
6307        let b = vector_unsigned_int([1, 2, 3, 4]);
6308        let d = unsafe { vec_find_any_eq_or_0_idx_cc(a, b, &mut c) };
6309        assert_eq!(c, 2);
6310        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6311
6312        // if no element of a matches any element of b with an equal value, and there is no element from a with a value of 0.
6313        let a = vector_unsigned_int([1, 2, 3, 4]);
6314        let b = vector_unsigned_int([5, 6, 7, 8]);
6315        let d = unsafe { vec_find_any_eq_or_0_idx_cc(a, b, &mut c) };
6316        assert_eq!(c, 3);
6317        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6318    }
6319
6320    #[simd_test(enable = "vector")]
6321    fn test_vec_find_any_ne_or_0_idx_cc() {
6322        let mut c = 0i32;
6323
6324        // if no element of a matches any element of b with a not equal value, and there is at least one element from a with a value of 0.
6325        let a = vector_unsigned_int([0, 1, 2, 3]);
6326        let b = vector_unsigned_int([4, 1, 2, 3]);
6327        let d = unsafe { vec_find_any_ne_or_0_idx_cc(a, b, &mut c) };
6328        assert_eq!(c, 0);
6329        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6330
6331        // if at least one element of a matches any element of b with a not equal value, and no elements of a with a value of 0.
6332        let a = vector_unsigned_int([4, 2, 3, 4]);
6333        let b = vector_unsigned_int([4, 5, 6, 7]);
6334        let d = unsafe { vec_find_any_ne_or_0_idx_cc(a, b, &mut c) };
6335        assert_eq!(c, 1);
6336        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6337
6338        // if at least one element of a matches any element of b with a not equal value, and there is at least one element from a has a value of 0.
6339        let a = vector_unsigned_int([1, 0, 1, 1]);
6340        let b = vector_unsigned_int([4, 5, 6, 7]);
6341        let d = unsafe { vec_find_any_ne_or_0_idx_cc(a, b, &mut c) };
6342        assert_eq!(c, 2);
6343        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6344
6345        // if no element of a matches any element of b with a not equal value, and there is no element from a with a value of 0.
6346        let a = vector_unsigned_int([4, 4, 4, 4]);
6347        let b = vector_unsigned_int([4, 5, 6, 7]);
6348        let d = unsafe { vec_find_any_ne_or_0_idx_cc(a, b, &mut c) };
6349        assert_eq!(c, 3);
6350        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6351    }
6352
6353    #[simd_test(enable = "vector")]
6354    fn test_vector_load() {
6355        let expected = [0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD];
6356
6357        let source: [u32; 8] = [
6358            0xAAAA_AAAA,
6359            0xBBBB_BBBB,
6360            0xCCCC_CCCC,
6361            0xDDDD_DDDD,
6362            0,
6363            0,
6364            0,
6365            0,
6366        ];
6367        assert_eq!(
6368            unsafe { vec_xl::<vector_unsigned_int>(0, source.as_ptr()) }.as_array(),
6369            &expected
6370        );
6371
6372        // offset is in bytes
6373        let source: [u32; 8] = [
6374            0x0000_AAAA,
6375            0xAAAA_BBBB,
6376            0xBBBB_CCCC,
6377            0xCCCC_DDDD,
6378            0xDDDD_0000,
6379            0,
6380            0,
6381            0,
6382        ];
6383        assert_eq!(
6384            unsafe { vec_xl::<vector_unsigned_int>(2, source.as_ptr()) }.as_array(),
6385            &expected
6386        );
6387    }
6388
6389    #[simd_test(enable = "vector")]
6390    fn test_vector_store() {
6391        let vec = vector_unsigned_int([0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD]);
6392
6393        let mut dest = [0u32; 8];
6394        unsafe { vec_xst(vec, 0, dest.as_mut_ptr()) };
6395        assert_eq!(
6396            dest,
6397            [
6398                0xAAAA_AAAA,
6399                0xBBBB_BBBB,
6400                0xCCCC_CCCC,
6401                0xDDDD_DDDD,
6402                0,
6403                0,
6404                0,
6405                0
6406            ]
6407        );
6408
6409        // offset is in bytes
6410        let mut dest = [0u32; 8];
6411        unsafe { vec_xst(vec, 2, dest.as_mut_ptr()) };
6412        assert_eq!(
6413            dest,
6414            [
6415                0x0000_AAAA,
6416                0xAAAA_BBBB,
6417                0xBBBB_CCCC,
6418                0xCCCC_DDDD,
6419                0xDDDD_0000,
6420                0,
6421                0,
6422                0,
6423            ]
6424        );
6425    }
6426
6427    #[simd_test(enable = "vector")]
6428    fn test_vector_lcbb() {
6429        #[repr(align(64))]
6430        struct Align64<T>(T);
6431
6432        static ARRAY: Align64<[u8; 128]> = Align64([0; 128]);
6433
6434        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[64..].as_ptr()) }, 16);
6435        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[63..].as_ptr()) }, 1);
6436        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[56..].as_ptr()) }, 8);
6437        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[48..].as_ptr()) }, 16);
6438    }
6439
6440    test_vec_2! { test_vec_pack, vec_pack, i16x8, i16x8 -> i8x16,
6441        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6442        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6443        [0, 1, -1, 42, -1, 0, 48, -48, -1, 0, 57, -57, 0, 1, -1, 42]
6444    }
6445
6446    test_vec_2! { test_vec_packs, vec_packs, i16x8, i16x8 -> i8x16,
6447        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6448        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6449        [0, 1, -1, 42, 127, -128, 127, -128, 127, -128, 127, -128, 0, 1, -1, 42]
6450    }
6451
6452    test_vec_2! { test_vec_packsu_signed, vec_packsu, i16x8, i16x8 -> u8x16,
6453        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6454        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6455        [0, 1, 0, 42, 255, 0, 255, 0, 255, 0, 255, 0, 0, 1, 0, 42]
6456    }
6457
6458    test_vec_2! { test_vec_packsu_unsigned, vec_packsu, u16x8, u16x8 -> u8x16,
6459        [65535, 32768, 1234, 5678, 16, 8, 4, 2],
6460        [30000, 25000, 20000, 15000, 31, 63, 127, 255],
6461        [255, 255, 255, 255, 16, 8, 4, 2, 255, 255, 255, 255, 31, 63, 127, 255]
6462    }
6463
6464    test_vec_2! { test_vec_rl, vec_rl, u32x4,
6465        [0x12345678, 0x9ABCDEF0, 0x0F0F0F0F, 0x12345678],
6466        [4, 8, 12, 68],
6467        [0x23456781, 0xBCDEF09A, 0xF0F0F0F0, 0x23456781]
6468    }
6469
6470    test_vec_1! { test_vec_unpackh_i, vec_unpackh, i16x8 -> i32x4,
6471        [0x1234, -2, 0x0F0F, -32768, 0, 0, 0, 0],
6472        [0x1234, -2, 0x0F0F, -32768]
6473    }
6474
6475    test_vec_1! { test_vec_unpackh_u, vec_unpackh, u16x8 -> u32x4,
6476        [0x1234, 0xFFFF, 0x0F0F, 0x8000, 0, 0, 0, 0],
6477        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6478    }
6479
6480    test_vec_1! { test_vec_unpackl_i, vec_unpackl, i16x8 -> i32x4,
6481        [0, 0, 0, 0, 0x1234, -2, 0x0F0F, -32768],
6482        [0x1234, -2, 0x0F0F, -32768]
6483    }
6484
6485    test_vec_1! { test_vec_unpackl_u, vec_unpackl, u16x8 -> u32x4,
6486        [0, 0, 0, 0, 0x1234, 0xFFFF, 0x0F0F, 0x8000],
6487        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6488    }
6489
6490    test_vec_2! { test_vec_avg, vec_avg, u32x4,
6491        [2, 1, u32::MAX, 0],
6492        [4, 2, 2, 0],
6493        [3, (1u32 + 2).div_ceil(2), (u32::MAX as u64 + 2u64).div_ceil(2) as u32, 0]
6494    }
6495
6496    test_vec_2! { test_vec_checksum, vec_checksum, u32x4,
6497        [1, 2, 3, u32::MAX],
6498        [5, 6, 7, 8],
6499        [0, 12, 0, 0]
6500    }
6501
6502    test_vec_2! { test_vec_add_u128, vec_add_u128, u8x16,
6503        [0x01, 0x05, 0x0F, 0x1A, 0x2F, 0x3F, 0x50, 0x65,
6504                              0x7A, 0x8F, 0x9A, 0xAD, 0xB0, 0xC3, 0xD5, 0xE8],
6505        [0xF0, 0xEF, 0xC3, 0xB1, 0x92, 0x71, 0x5A, 0x43,
6506                              0x3B, 0x29, 0x13, 0x04, 0xD7, 0xA1, 0x8C, 0x76],
6507        [0xF1, 0xF4, 0xD2, 0xCB, 0xC1, 0xB0, 0xAA, 0xA8, 0xB5, 0xB8, 0xAD, 0xB2, 0x88, 0x65, 0x62, 0x5E]
6508    }
6509
6510    #[simd_test(enable = "vector")]
6511    fn test_vec_addc_u128() {
6512        unsafe {
6513            let a = u128::MAX;
6514            let b = 1u128;
6515
6516            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6517            assert!(a.checked_add(b).is_none());
6518            assert_eq!(d, 1);
6519
6520            let a = 1u128;
6521            let b = 1u128;
6522
6523            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6524            assert!(a.checked_add(b).is_some());
6525            assert_eq!(d, 0);
6526        }
6527    }
6528
6529    #[simd_test(enable = "vector")]
6530    fn test_vec_subc_u128() {
6531        unsafe {
6532            let a = 0u128;
6533            let b = 1u128;
6534
6535            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6536            assert!(a.checked_sub(b).is_none());
6537            assert_eq!(d, 0);
6538
6539            let a = 1u128;
6540            let b = 1u128;
6541
6542            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6543            assert!(a.checked_sub(b).is_some());
6544            assert_eq!(d, 1);
6545        }
6546    }
6547
6548    test_vec_2! { test_vec_mule_u, vec_mule, u16x8, u16x8 -> u32x4,
6549        [0xFFFF, 0, 2, 0, 2, 0, 1, 0],
6550        [0xFFFF, 0, 4, 0, 0xFFFF, 0, 2, 0],
6551        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6552    }
6553
6554    test_vec_2! { test_vec_mule_i, vec_mule, i16x8, i16x8 -> i32x4,
6555        [i16::MIN, 0, -2, 0, 2, 0, 1, 0],
6556        [i16::MIN, 0, 4, 0, i16::MAX, 0, 2, 0],
6557        [0x4000_0000, -8, 0xFFFE, 2]
6558    }
6559
6560    test_vec_2! { test_vec_mulo_u, vec_mulo, u16x8, u16x8 -> u32x4,
6561        [0, 0xFFFF, 0, 2, 0, 2, 0, 1],
6562        [0, 0xFFFF, 0, 4, 0, 0xFFFF, 0, 2],
6563        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6564    }
6565
6566    test_vec_2! { test_vec_mulo_i, vec_mulo, i16x8, i16x8 -> i32x4,
6567        [0, i16::MIN, 0, -2, 0, 2, 0, 1],
6568        [0, i16::MIN, 0, 4, 0, i16::MAX, 0, 2],
6569        [0x4000_0000, -8, 0xFFFE, 2]
6570    }
6571
6572    test_vec_2! { test_vec_mulh_u, vec_mulh, u32x4, u32x4 -> u32x4,
6573        [u32::MAX, 2, 2, 1],
6574        [u32::MAX, 4, u32::MAX, 2],
6575        [u32::MAX - 1, 0, 1, 0]
6576    }
6577
6578    test_vec_2! { test_vec_mulh_i, vec_mulh, i32x4, i32x4 -> i32x4,
6579        [i32::MIN, -2, 2, 1],
6580        [i32::MIN, 4, i32::MAX, 2],
6581        [0x4000_0000, -1, 0, 0]
6582    }
6583
6584    test_vec_2! { test_vec_gfmsum_1, vec_gfmsum, u16x8, u16x8 -> u32x4,
6585        [0x1234, 0x5678, 0x9ABC, 0xDEF0, 0x1357, 0x2468, 0xACE0, 0xBDF0],
6586        [0xFFFF, 0x0001, 0x8000, 0x7FFF, 0xAAAA, 0x5555, 0x1234, 0x5678],
6587        [0xE13A794, 0x68764A50, 0x94AA3E, 0x2C93F300]
6588    }
6589
6590    test_vec_2! { test_vec_gfmsum_2, vec_gfmsum, u16x8, u16x8 -> u32x4,
6591        [0x0000, 0xFFFF, 0xAAAA, 0x5555, 0x1234, 0x5678, 0x9ABC, 0xDEF0],
6592        [0xFFFF, 0x0000, 0x5555, 0xAAAA, 0x0001, 0x8000, 0x7FFF, 0x1357],
6593        [0, 0, 0x2B3C1234, 0x3781D244]
6594    }
6595
6596    #[simd_test(enable = "vector")]
6597    fn test_vec_gfmsum_128() {
6598        let a = vector_unsigned_long_long([1, 2]);
6599        let b = vector_unsigned_long_long([3, 4]);
6600
6601        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6602        assert_eq!(d, 11);
6603
6604        let a = vector_unsigned_long_long([0x0101010101010101, 0x0202020202020202]);
6605        let b = vector_unsigned_long_long([0x0404040404040404, 0x0505050505050505]);
6606
6607        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6608        assert_eq!(d, 0xE000E000E000E000E000E000E000E);
6609    }
6610
6611    #[simd_test(enable = "vector-enhancements-1")]
6612    fn test_vec_bperm_u128() {
6613        let a = vector_unsigned_char([65, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]);
6614        let b = vector_unsigned_char([
6615            0, 0, 0, 0, 1, 1, 1, 1, 128, 128, 128, 128, 255, 255, 255, 255,
6616        ]);
6617        let d = unsafe { vec_bperm_u128(a, b) };
6618        assert_eq!(d.as_array(), &[0xF00, 0]);
6619    }
6620
6621    #[simd_test(enable = "vector")]
6622    fn test_vec_sel() {
6623        let a = vector_signed_int([1, 2, 3, 4]);
6624        let b = vector_signed_int([5, 6, 7, 8]);
6625
6626        let e = vector_unsigned_int([9, 10, 11, 12]);
6627        let f = vector_unsigned_int([9, 9, 11, 11]);
6628
6629        let c: vector_bool_int = unsafe { simd_eq(e, f) };
6630        assert_eq!(c.as_array(), &[!0, 0, !0, 0]);
6631        let d: vector_signed_int = unsafe { vec_sel(a, b, c) };
6632        assert_eq!(d.as_array(), &[5, 2, 7, 4]);
6633    }
6634
6635    #[simd_test(enable = "vector")]
6636    fn test_vec_gather_element() {
6637        let a1: [u32; 10] = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19];
6638        let a2: [u32; 10] = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29];
6639
6640        let v1 = vector_unsigned_int([1, 2, 3, 4]);
6641        let v2 = vector_unsigned_int([1, 2, 3, 4]);
6642
6643        let sizeof_int = core::mem::size_of::<u32>() as u32;
6644        let v3 = vector_unsigned_int([
6645            5 * sizeof_int,
6646            8 * sizeof_int,
6647            9 * sizeof_int,
6648            6 * sizeof_int,
6649        ]);
6650
6651        unsafe {
6652            let d1 = vec_gather_element::<_, 0>(v1, v3, a1.as_ptr());
6653            assert_eq!(d1.as_array(), &[15, 2, 3, 4]);
6654            let d2 = vec_gather_element::<_, 0>(v2, v3, a2.as_ptr());
6655            assert_eq!(d2.as_array(), &[25, 2, 3, 4]);
6656        }
6657    }
6658
6659    #[simd_test(enable = "vector")]
6660    fn test_vec_fp_test_data_class() {
6661        let mut cc = 42;
6662
6663        let v1 = vector_double([0.0, f64::NAN]);
6664        let v2 = vector_double([f64::INFINITY, 1.0]);
6665        let v3 = vector_double([1.0, 2.0]);
6666
6667        unsafe {
6668            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_ZERO>(v1, &mut cc);
6669            assert_eq!(cc, 1);
6670            assert_eq!(d.as_array(), &[!0, 0]);
6671
6672            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NAN>(v1, &mut cc);
6673            assert_eq!(cc, 1);
6674            assert_eq!(d.as_array(), &[0, !0]);
6675
6676            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY>(v2, &mut cc);
6677            assert_eq!(cc, 1);
6678            assert_eq!(d.as_array(), &[!0, 0]);
6679
6680            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY_N>(v2, &mut cc);
6681            assert_eq!(cc, 3);
6682            assert_eq!(d.as_array(), &[0, 0]);
6683
6684            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v2, &mut cc);
6685            assert_eq!(cc, 1);
6686            assert_eq!(d.as_array(), &[0, !0]);
6687
6688            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v3, &mut cc);
6689            assert_eq!(cc, 0);
6690            assert_eq!(d.as_array(), &[!0, !0]);
6691        }
6692    }
6693
6694    #[simd_test(enable = "vector")]
6695    fn test_vec_fp_any_all_nan_numeric() {
6696        unsafe {
6697            assert_eq!(
6698                vec_all_nan(vector_double([f64::NAN, f64::NAN])),
6699                i32::from(true)
6700            );
6701            assert_eq!(
6702                vec_all_nan(vector_double([f64::NAN, 1.0])),
6703                i32::from(false)
6704            );
6705            assert_eq!(vec_all_nan(vector_double([0.0, 1.0])), i32::from(false));
6706
6707            assert_eq!(
6708                vec_any_nan(vector_double([f64::NAN, f64::NAN])),
6709                i32::from(true)
6710            );
6711            assert_eq!(vec_any_nan(vector_double([f64::NAN, 1.0])), i32::from(true));
6712            assert_eq!(vec_any_nan(vector_double([0.0, 1.0])), i32::from(false));
6713
6714            assert_eq!(
6715                vec_all_numeric(vector_double([f64::NAN, f64::NAN])),
6716                i32::from(false)
6717            );
6718            assert_eq!(
6719                vec_all_numeric(vector_double([f64::NAN, 1.0])),
6720                i32::from(false)
6721            );
6722            assert_eq!(vec_all_numeric(vector_double([0.0, 1.0])), i32::from(true));
6723
6724            assert_eq!(
6725                vec_any_numeric(vector_double([f64::NAN, f64::NAN])),
6726                i32::from(false)
6727            );
6728            assert_eq!(
6729                vec_any_numeric(vector_double([f64::NAN, 1.0])),
6730                i32::from(true)
6731            );
6732            assert_eq!(vec_any_numeric(vector_double([0.0, 1.0])), i32::from(true));
6733
6734            // "numeric" means "not NaN". infinities are numeric
6735            assert_eq!(
6736                vec_all_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
6737                i32::from(true)
6738            );
6739            assert_eq!(
6740                vec_any_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
6741                i32::from(true)
6742            );
6743        }
6744    }
6745
6746    #[simd_test(enable = "vector")]
6747    fn test_vec_test_mask() {
6748        unsafe {
6749            let v = vector_unsigned_long_long([0xFF00FF00FF00FF00; 2]);
6750            let m = vector_unsigned_long_long([0x0000FF000000FF00; 2]);
6751            assert_eq!(vec_test_mask(v, m), 3);
6752
6753            let v = vector_unsigned_long_long([u64::MAX; 2]);
6754            let m = vector_unsigned_long_long([0; 2]);
6755            assert_eq!(vec_test_mask(v, m), 0);
6756
6757            let v = vector_unsigned_long_long([0; 2]);
6758            let m = vector_unsigned_long_long([u64::MAX; 2]);
6759            assert_eq!(vec_test_mask(v, m), 0);
6760
6761            let v = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
6762            let m = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
6763            assert_eq!(vec_test_mask(v, m), 3);
6764        }
6765    }
6766
6767    #[simd_test(enable = "vector-enhancements-2")]
6768    fn test_vec_search_string_cc() {
6769        unsafe {
6770            let b = vector_unsigned_char(*b"ABCD------------");
6771            let c = vector_unsigned_char([4; 16]);
6772            let mut d = 0i32;
6773
6774            let haystack = vector_unsigned_char(*b"__ABCD__________");
6775            let result = vec_search_string_cc(haystack, b, c, &mut d);
6776            assert_eq!(result.as_array()[7], 2);
6777            assert_eq!(d, 2);
6778
6779            let haystack = vector_unsigned_char(*b"___ABCD_________");
6780            let result = vec_search_string_cc(haystack, b, c, &mut d);
6781            assert_eq!(result.as_array()[7], 3);
6782            assert_eq!(d, 2);
6783
6784            let haystack = vector_unsigned_char(*b"________________");
6785            let result = vec_search_string_cc(haystack, b, c, &mut d);
6786            assert_eq!(result.as_array()[7], 16);
6787            assert_eq!(d, 0);
6788
6789            let haystack = vector_unsigned_char(*b"______\0_________");
6790            let result = vec_search_string_cc(haystack, b, c, &mut d);
6791            assert_eq!(result.as_array()[7], 16);
6792            assert_eq!(d, 0);
6793
6794            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
6795            let result = vec_search_string_cc(haystack, b, c, &mut d);
6796            assert_eq!(result.as_array()[7], 9);
6797            assert_eq!(d, 2);
6798        }
6799    }
6800
6801    #[simd_test(enable = "vector-enhancements-2")]
6802    fn test_vec_search_string_until_zero_cc() {
6803        unsafe {
6804            let b = vector_unsigned_char(*b"ABCD\0\0\0\0\0\0\0\0\0\0\0\0");
6805            let c = vector_unsigned_char([16; 16]);
6806            let mut d = 0i32;
6807
6808            let haystack = vector_unsigned_char(*b"__ABCD__________");
6809            let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d);
6810            assert_eq!(result.as_array()[7], 2);
6811            assert_eq!(d, 2);
6812
6813            let haystack = vector_unsigned_char(*b"___ABCD_________");
6814            let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d);
6815            assert_eq!(result.as_array()[7], 3);
6816            assert_eq!(d, 2);
6817
6818            let haystack = vector_unsigned_char(*b"________________");
6819            let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d);
6820            assert_eq!(result.as_array()[7], 16);
6821            assert_eq!(d, 0);
6822
6823            let haystack = vector_unsigned_char(*b"______\0_________");
6824            let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d);
6825            assert_eq!(result.as_array()[7], 16);
6826            assert_eq!(d, 1);
6827
6828            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
6829            let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d);
6830            assert_eq!(result.as_array()[7], 16);
6831            assert_eq!(d, 1);
6832        }
6833    }
6834
6835    #[simd_test(enable = "vector")]
6836    fn test_vec_doublee() {
6837        unsafe {
6838            let v = vector_float([1.0, 2.0, 3.0, 4.0]);
6839            assert_eq!(vec_doublee(v).as_array(), &[1.0, 3.0]);
6840
6841            let v = vector_float([f32::NAN, 2.0, f32::INFINITY, 4.0]);
6842            let d = vec_doublee(v);
6843            assert!(d.as_array()[0].is_nan());
6844            assert_eq!(d.as_array()[1], f64::INFINITY);
6845        }
6846    }
6847
6848    #[simd_test(enable = "vector")]
6849    fn test_vec_floate() {
6850        // NOTE: indices 1 and 3 can have an arbitrary value. With the C version
6851        // these are poison values, our version initializes the memory but its
6852        // value still should not be relied upon by application code.
6853        unsafe {
6854            let v = vector_double([1.0, 2.0]);
6855            let d = vec_floate(v);
6856            assert_eq!(d.as_array()[0], 1.0);
6857            assert_eq!(d.as_array()[2], 2.0);
6858
6859            let v = vector_double([f64::NAN, f64::INFINITY]);
6860            let d = vec_floate(v);
6861            assert!(d.as_array()[0].is_nan());
6862            assert_eq!(d.as_array()[2], f32::INFINITY);
6863
6864            let v = vector_double([f64::MIN, f64::MAX]);
6865            let d = vec_floate(v);
6866            assert_eq!(d.as_array()[0], f64::MIN as f32);
6867            assert_eq!(d.as_array()[2], f64::MAX as f32);
6868        }
6869    }
6870
6871    #[simd_test(enable = "vector")]
6872    fn test_vec_extend_s64() {
6873        unsafe {
6874            let v = vector_signed_char([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
6875            assert_eq!(vec_extend_s64(v).as_array(), &[7, 15]);
6876
6877            let v = vector_signed_short([0, 1, 2, 3, 4, 5, 6, 7]);
6878            assert_eq!(vec_extend_s64(v).as_array(), &[3, 7]);
6879
6880            let v = vector_signed_int([0, 1, 2, 3]);
6881            assert_eq!(vec_extend_s64(v).as_array(), &[1, 3]);
6882        }
6883    }
6884
6885    #[simd_test(enable = "vector")]
6886    fn test_vec_signed() {
6887        unsafe {
6888            let v = vector_float([1.0, 2.5, -2.5, -0.0]);
6889            assert_eq!(vec_signed(v).as_array(), &[1, 2, -2, 0]);
6890
6891            let v = vector_double([2.5, -2.5]);
6892            assert_eq!(vec_signed(v).as_array(), &[2, -2]);
6893        }
6894    }
6895
6896    #[simd_test(enable = "vector")]
6897    fn test_vec_unsigned() {
6898        // NOTE: converting a negative floating point value is UB!
6899        unsafe {
6900            let v = vector_float([1.0, 2.5, 3.5, 0.0]);
6901            assert_eq!(vec_unsigned(v).as_array(), &[1, 2, 3, 0]);
6902
6903            let v = vector_double([2.5, 3.5]);
6904            assert_eq!(vec_unsigned(v).as_array(), &[2, 3]);
6905        }
6906    }
6907
6908    #[simd_test(enable = "vector")]
6909    fn test_vec_cp_until_zero() {
6910        unsafe {
6911            let v = vector_signed_int([1, 2, 3, 4]);
6912            let d = vec_cp_until_zero(v);
6913            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
6914
6915            let v = vector_signed_int([1, 2, 0, 4]);
6916            let d = vec_cp_until_zero(v);
6917            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
6918        }
6919    }
6920
6921    #[simd_test(enable = "vector")]
6922    fn test_vec_cp_until_zero_cc() {
6923        let mut cc = 0;
6924        unsafe {
6925            let v = vector_signed_int([1, 2, 3, 4]);
6926            let d = vec_cp_until_zero_cc(v, &mut cc);
6927            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
6928            assert_eq!(cc, 3);
6929
6930            let v = vector_signed_int([1, 2, 0, 4]);
6931            let d = vec_cp_until_zero_cc(v, &mut cc);
6932            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
6933            assert_eq!(cc, 0);
6934        }
6935    }
6936
6937    #[simd_test(enable = "vector-enhancements-1")]
6938    fn test_vec_msum_u128() {
6939        let a = vector_unsigned_long_long([1, 2]);
6940        let b = vector_unsigned_long_long([3, 4]);
6941
6942        unsafe {
6943            let c: vector_unsigned_char = transmute(100u128);
6944
6945            let d: u128 = transmute(vec_msum_u128::<0>(a, b, c));
6946            assert_eq!(d, (1 * 3) + (2 * 4) + 100);
6947
6948            let d: u128 = transmute(vec_msum_u128::<4>(a, b, c));
6949            assert_eq!(d, (1 * 3) + (2 * 4) * 2 + 100);
6950
6951            let d: u128 = transmute(vec_msum_u128::<8>(a, b, c));
6952            assert_eq!(d, (1 * 3) * 2 + (2 * 4) + 100);
6953
6954            let d: u128 = transmute(vec_msum_u128::<12>(a, b, c));
6955            assert_eq!(d, (1 * 3) * 2 + (2 * 4) * 2 + 100);
6956        }
6957    }
6958
6959    #[simd_test(enable = "vector")]
6960    fn test_vec_sld() {
6961        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
6962        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
6963
6964        unsafe {
6965            let d = vec_sld::<_, 4>(a, b);
6966            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
6967        }
6968    }
6969
6970    #[simd_test(enable = "vector")]
6971    fn test_vec_sldw() {
6972        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
6973        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
6974
6975        unsafe {
6976            let d = vec_sldw::<_, 1>(a, b);
6977            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
6978        }
6979    }
6980
6981    #[simd_test(enable = "vector-enhancements-2")]
6982    fn test_vec_sldb() {
6983        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
6984        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
6985
6986        unsafe {
6987            let d = vec_sldb::<_, 4>(a, b);
6988            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAB]);
6989        }
6990    }
6991
6992    #[simd_test(enable = "vector-enhancements-2")]
6993    fn test_vec_srdb() {
6994        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
6995        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
6996
6997        unsafe {
6998            let d = vec_srdb::<_, 4>(a, b);
6999            assert_eq!(d.as_array(), &[0xABBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7000        }
7001    }
7002
7003    const GT: u32 = 0x20000000;
7004    const LT: u32 = 0x40000000;
7005    const EQ: u32 = 0x80000000;
7006
7007    #[simd_test(enable = "vector")]
7008    fn test_vec_cmprg() {
7009        let a = vector_unsigned_int([11, 22, 33, 44]);
7010        let b = vector_unsigned_int([10, 20, 30, 40]);
7011
7012        let c = vector_unsigned_int([GT, LT, GT, LT]);
7013        let d = unsafe { vec_cmprg(a, b, c) };
7014        assert_eq!(d.as_array(), &[!0, 0, !0, 0]);
7015
7016        let c = vector_unsigned_int([GT, LT, 0, 0]);
7017        let d = unsafe { vec_cmprg(a, b, c) };
7018        assert_eq!(d.as_array(), &[!0, 0, 0, 0]);
7019
7020        let a = vector_unsigned_int([11, 22, 33, 30]);
7021        let b = vector_unsigned_int([10, 20, 30, 30]);
7022
7023        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7024        let d = unsafe { vec_cmprg(a, b, c) };
7025        assert_eq!(d.as_array(), &[!0, 0, 0, !0]);
7026    }
7027
7028    #[simd_test(enable = "vector")]
7029    fn test_vec_cmpnrg() {
7030        let a = vector_unsigned_int([11, 22, 33, 44]);
7031        let b = vector_unsigned_int([10, 20, 30, 40]);
7032
7033        let c = vector_unsigned_int([GT, LT, GT, LT]);
7034        let d = unsafe { vec_cmpnrg(a, b, c) };
7035        assert_eq!(d.as_array(), &[0, !0, 0, !0]);
7036
7037        let c = vector_unsigned_int([GT, LT, 0, 0]);
7038        let d = unsafe { vec_cmpnrg(a, b, c) };
7039        assert_eq!(d.as_array(), &[0, !0, !0, !0]);
7040
7041        let a = vector_unsigned_int([11, 22, 33, 30]);
7042        let b = vector_unsigned_int([10, 20, 30, 30]);
7043
7044        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7045        let d = unsafe { vec_cmpnrg(a, b, c) };
7046        assert_eq!(d.as_array(), &[0, !0, !0, 0]);
7047    }
7048
7049    #[simd_test(enable = "vector")]
7050    fn test_vec_cmprg_idx() {
7051        let a = vector_unsigned_int([1, 11, 22, 33]);
7052        let b = vector_unsigned_int([10, 20, 30, 40]);
7053
7054        let c = vector_unsigned_int([GT, LT, GT, LT]);
7055        let d = unsafe { vec_cmprg_idx(a, b, c) };
7056        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7057    }
7058
7059    #[simd_test(enable = "vector")]
7060    fn test_vec_cmpnrg_idx() {
7061        let a = vector_unsigned_int([1, 11, 22, 33]);
7062        let b = vector_unsigned_int([10, 20, 30, 40]);
7063
7064        let c = vector_unsigned_int([GT, LT, GT, LT]);
7065        let d = unsafe { vec_cmpnrg_idx(a, b, c) };
7066        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
7067    }
7068
7069    #[simd_test(enable = "vector")]
7070    fn test_vec_cmprg_or_0_idx() {
7071        let a = vector_unsigned_int([1, 0, 22, 33]);
7072        let b = vector_unsigned_int([10, 20, 30, 40]);
7073
7074        let c = vector_unsigned_int([GT, LT, GT, LT]);
7075        let d = unsafe { vec_cmprg_or_0_idx(a, b, c) };
7076        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7077    }
7078
7079    #[simd_test(enable = "vector")]
7080    fn test_vec_cmpnrg_or_0_idx() {
7081        let a = vector_unsigned_int([11, 33, 0, 22]);
7082        let b = vector_unsigned_int([10, 20, 30, 40]);
7083
7084        let c = vector_unsigned_int([GT, LT, GT, LT]);
7085        let d = unsafe { vec_cmpnrg_or_0_idx(a, b, c) };
7086        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
7087    }
7088
7089    test_vec_2! { test_vec_cmpgt, vec_cmpgt, f32x4, f32x4 -> i32x4,
7090        [1.0, f32::NAN, f32::NAN, 3.14],
7091        [2.0, f32::NAN, 5.0, 2.0],
7092        [0, 0, 0, !0]
7093    }
7094
7095    test_vec_2! { test_vec_cmpge, vec_cmpge, f32x4, f32x4 -> i32x4,
7096        [1.0, f32::NAN, f32::NAN, 3.14],
7097        [1.0, f32::NAN, 5.0, 2.0],
7098        [!0, 0, 0, !0]
7099    }
7100
7101    test_vec_2! { test_vec_cmplt, vec_cmplt, f32x4, f32x4 -> i32x4,
7102        [1.0, f32::NAN, f32::NAN, 2.0],
7103        [2.0, f32::NAN, 5.0, 2.0],
7104        [!0, 0, 0, 0]
7105    }
7106
7107    test_vec_2! { test_vec_cmple, vec_cmple, f32x4, f32x4 -> i32x4,
7108        [1.0, f32::NAN, f32::NAN, 2.0],
7109        [1.0, f32::NAN, 5.0, 3.14],
7110        [!0, 0, 0, !0]
7111    }
7112
7113    test_vec_2! { test_vec_cmpeq, vec_cmpeq, f32x4, f32x4 -> i32x4,
7114        [1.0, f32::NAN, f32::NAN, 2.0],
7115        [1.0, f32::NAN, 5.0, 3.14],
7116        [!0, 0, 0, 0]
7117    }
7118
7119    test_vec_2! { test_vec_cmpne, vec_cmpne, f32x4, f32x4 -> i32x4,
7120        [1.0, f32::NAN, f32::NAN, 2.0],
7121        [1.0, f32::NAN, 5.0, 3.14],
7122        [0, !0, !0, !0]
7123    }
7124
7125    #[simd_test(enable = "vector")]
7126    fn test_vec_meadd() {
7127        let a = vector_unsigned_short([1, 0, 2, 0, 3, 0, 4, 0]);
7128        let b = vector_unsigned_short([5, 0, 6, 0, 7, 0, 8, 0]);
7129        let c = vector_unsigned_int([2, 2, 2, 2]);
7130
7131        let d = unsafe { vec_meadd(a, b, c) };
7132        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7133
7134        let a = vector_signed_short([1, 0, 2, 0, 3, 0, 4, 0]);
7135        let b = vector_signed_short([5, 0, 6, 0, 7, 0, 8, 0]);
7136        let c = vector_signed_int([2, -2, 2, -2]);
7137
7138        let d = unsafe { vec_meadd(a, b, c) };
7139        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7140    }
7141
7142    #[simd_test(enable = "vector")]
7143    fn test_vec_moadd() {
7144        let a = vector_unsigned_short([0, 1, 0, 2, 0, 3, 0, 4]);
7145        let b = vector_unsigned_short([0, 5, 0, 6, 0, 7, 0, 8]);
7146        let c = vector_unsigned_int([2, 2, 2, 2]);
7147
7148        let d = unsafe { vec_moadd(a, b, c) };
7149        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7150
7151        let a = vector_signed_short([0, 1, 0, 2, 0, 3, 0, 4]);
7152        let b = vector_signed_short([0, 5, 0, 6, 0, 7, 0, 8]);
7153        let c = vector_signed_int([2, -2, 2, -2]);
7154
7155        let d = unsafe { vec_moadd(a, b, c) };
7156        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7157    }
7158
7159    #[simd_test(enable = "vector")]
7160    fn test_vec_mhadd() {
7161        let a = vector_unsigned_int([1, 2, 3, 4]);
7162        let b = vector_unsigned_int([5, 6, 7, 8]);
7163        let c = vector_unsigned_int([u32::MAX; 4]);
7164
7165        let d = unsafe { vec_mhadd(a, b, c) };
7166        assert_eq!(d.as_array(), &[1, 1, 1, 1]);
7167
7168        let a = vector_signed_int([-1, -2, -3, -4]);
7169        let b = vector_signed_int([5, 6, 7, 8]);
7170        let c = vector_signed_int([i32::MIN; 4]);
7171
7172        let d = unsafe { vec_mhadd(a, b, c) };
7173        assert_eq!(d.as_array(), &[-1, -1, -1, -1]);
7174    }
7175
7176    #[simd_test(enable = "vector")]
7177    fn test_vec_mladd() {
7178        let a = vector_unsigned_int([1, 2, 3, 4]);
7179        let b = vector_unsigned_int([5, 6, 7, 8]);
7180        let c = vector_unsigned_int([2, 2, 2, 2]);
7181
7182        let d = unsafe { vec_mladd(a, b, c) };
7183        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7184
7185        let a = vector_signed_int([-1, -2, -3, -4]);
7186        let b = vector_signed_int([5, 6, 7, 8]);
7187        let c = vector_signed_int([2, 2, 2, 2]);
7188
7189        let d = unsafe { vec_mladd(a, b, c) };
7190        assert_eq!(d.as_array(), &[-3, -10, -19, -30]);
7191    }
7192}