2
* Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5
* This code is free software; you can redistribute it and/or modify it
6
* under the terms of the GNU General Public License version 2 only, as
7
* published by the Free Software Foundation.
9
* This code is distributed in the hope that it will be useful, but WITHOUT
10
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12
* version 2 for more details (a copy is included in the LICENSE file that
13
* accompanied this code).
15
* You should have received a copy of the GNU General Public License version
16
* 2 along with this work; if not, write to the Free Software Foundation,
17
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20
* or visit www.oracle.com if you need additional information or have any
25
#include "precompiled.hpp"
26
#include "ci/ciSymbols.hpp"
27
#include "classfile/vmSymbols.hpp"
28
#include "opto/library_call.hpp"
29
#include "opto/runtime.hpp"
30
#include "opto/vectornode.hpp"
31
#include "prims/vectorSupport.hpp"
32
#include "runtime/stubRoutines.hpp"
35
static bool is_vector(ciKlass* klass) {
36
return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
39
static bool check_vbox(const TypeInstPtr* vbox_type) {
40
assert(vbox_type->klass_is_exact(), "");
42
ciInstanceKlass* ik = vbox_type->instance_klass();
43
assert(is_vector(ik), "not a vector");
45
ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46
assert(fd1 != nullptr, "element type info is missing");
48
ciConstant val1 = fd1->constant_value();
49
BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50
assert(is_java_primitive(elem_bt), "element type info is missing");
52
ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53
assert(fd2 != nullptr, "vector length info is missing");
55
ciConstant val2 = fd2->constant_value();
56
assert(val2.as_int() > 0, "vector length info is missing");
62
#define log_if_needed(...) \
63
if (C->print_intrinsics()) { \
64
tty->print_cr(__VA_ARGS__); \
68
#define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
70
#define non_product_log_if_needed(...)
73
static bool is_vector_mask(ciKlass* klass) {
74
return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
77
static bool is_vector_shuffle(ciKlass* klass) {
78
return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
81
bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
82
VectorMaskUseType mask_use_type, bool has_scalar_args) {
83
bool is_supported = true;
85
// has_scalar_args flag is true only for non-constant scalar shift count,
86
// since in this case shift needs to be broadcasted.
87
if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
88
(has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
93
// Check if mask unboxing is supported, this is a two step process which first loads the contents
94
// of boolean array into vector followed by either lane expansion to match the lane size of masked
95
// vector operation or populate the predicate register.
96
if ((mask_use_type & VecMaskUseLoad) != 0) {
97
if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
98
!Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
99
non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
100
NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
105
if ((mask_use_type & VecMaskUsePred) != 0) {
106
if (!Matcher::has_predicated_vectors() ||
107
!Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
108
non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
109
NodeClassNames[opc], type2name(elem_bt), num_elem);
115
int lshiftopc, rshiftopc;
118
lshiftopc = Op_LShiftI;
119
rshiftopc = Op_URShiftB;
122
lshiftopc = Op_LShiftI;
123
rshiftopc = Op_URShiftS;
126
lshiftopc = Op_LShiftI;
127
rshiftopc = Op_URShiftI;
130
lshiftopc = Op_LShiftL;
131
rshiftopc = Op_URShiftL;
133
default: fatal("Unexpected type: %s", type2name(elem_bt));
135
int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
136
int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
138
arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
139
arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
140
arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
146
Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
147
assert(EnableVectorSupport, "");
149
PreserveReexecuteState preexecs(this);
150
jvms()->set_should_reexecute(true);
152
VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
153
set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
154
make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
155
set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
156
set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
157
Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
159
assert(check_vbox(vbox_type), "");
160
const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
161
VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
162
return gvn().transform(vbox);
165
Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
166
assert(EnableVectorSupport, "");
167
const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
168
if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
169
return nullptr; // arguments don't agree on vector shapes
171
if (vbox_type_v->maybe_null()) {
172
return nullptr; // no nulls are allowed
174
assert(check_vbox(vbox_type), "");
175
const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
176
Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
180
Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
181
assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
182
juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
183
Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
184
Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
185
return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
188
bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
189
// Check that the operation is valid.
191
non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
195
if (VectorNode::is_vector_rotate(sopc)) {
196
if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
197
non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
198
NodeClassNames[sopc], type2name(type), num_elem);
201
} else if (VectorNode::is_vector_integral_negate(sopc)) {
202
if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
203
non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
204
NodeClassNames[sopc], type2name(type), num_elem);
208
// Check that architecture supports this op-size-type combination.
209
if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
210
non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
211
NodeClassNames[sopc], type2name(type), num_elem);
214
assert(Matcher::match_rule_supported(sopc), "must be supported");
219
if (mask_use_type != VecMaskNotUsed) {
220
non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
221
NodeClassNames[sopc], type2name(type), num_elem);
226
if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
227
non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
228
NodeClassNames[sopc], type2name(type), num_elem);
234
if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
235
Matcher::supports_vector_variable_shifts() == false) {
236
log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
237
NodeClassNames[sopc], type2name(type), num_elem);
241
// Check if mask unboxing is supported, this is a two step process which first loads the contents
242
// of boolean array into vector followed by either lane expansion to match the lane size of masked
243
// vector operation or populate the predicate register.
244
if ((mask_use_type & VecMaskUseLoad) != 0) {
245
if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
246
!Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
247
non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
248
NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
253
// Check if mask boxing is supported, this is a two step process which first stores the contents
254
// of mask vector / predicate register into a boolean vector followed by vector store operation to
255
// transfer the contents to underlined storage of mask boxes which is a boolean array.
256
if ((mask_use_type & VecMaskUseStore) != 0) {
257
if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
258
!Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
259
non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
260
NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
265
if ((mask_use_type & VecMaskUsePred) != 0) {
266
bool is_supported = false;
267
if (Matcher::has_predicated_vectors()) {
268
if (VectorNode::is_vector_integral_negate(sopc)) {
269
is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
271
is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
274
is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
277
non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
278
NodeClassNames[sopc], type2name(type), num_elem);
286
static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
287
if (vec_klass->const_oop() == nullptr) {
288
return false; // uninitialized or some kind of unsafe access
290
assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
291
ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
292
return klass->is_initialized();
296
// <V extends Vector<E>,
297
// M extends VectorMask<E>,
299
// V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
300
// int length, V v, M m,
301
// UnaryOperation<V, M> defaultImpl)
305
// M extends VectorMask<E>,
307
// V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308
// int length, V v1, V v2, M m,
309
// BinaryOperation<V, M> defaultImpl)
312
// <V extends Vector<E>,
313
// M extends VectorMask<E>,
315
// V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316
// int length, V v1, V v2, V v3, M m,
317
// TernaryOperation<V, M> defaultImpl)
319
bool LibraryCallKit::inline_vector_nary_operation(int n) {
320
const TypeInt* opr = gvn().type(argument(0))->isa_int();
321
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323
const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
326
if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
327
!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
328
log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
329
NodeClassNames[argument(0)->Opcode()],
330
NodeClassNames[argument(1)->Opcode()],
331
NodeClassNames[argument(3)->Opcode()],
332
NodeClassNames[argument(4)->Opcode()]);
333
return false; // not enough info for intrinsification
336
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
337
if (!elem_type->is_primitive_type()) {
338
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
339
return false; // should be primitive type
341
if (!is_klass_initialized(vector_klass)) {
342
log_if_needed(" ** klass argument not initialized");
346
// "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
347
// is used to control this operation.
348
const Type* vmask_type = gvn().type(argument(n + 5));
349
bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
351
if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
352
log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
353
return false; // not enough info for intrinsification
356
if (!is_klass_initialized(mask_klass)) {
357
log_if_needed(" ** mask klass argument not initialized");
361
if (vmask_type->maybe_null()) {
362
log_if_needed(" ** null mask values are not allowed for masked op");
367
BasicType elem_bt = elem_type->basic_type();
368
int num_elem = vlen->get_con();
369
int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
370
int sopc = VectorNode::opcode(opc, elem_bt);
371
if ((opc != Op_CallLeafVector) && (sopc == 0)) {
372
log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
373
return false; // operation not supported
376
if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
377
log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
378
n, opc, num_elem, type2name(elem_bt));
382
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
383
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
385
if (is_vector_mask(vbox_klass)) {
386
assert(!is_masked_op, "mask operations do not need mask to control");
389
if (opc == Op_CallLeafVector) {
390
if (!UseVectorStubs) {
391
log_if_needed(" ** vector stubs support is disabled");
394
if (!Matcher::supports_vector_calling_convention()) {
395
log_if_needed(" ** no vector calling conventions supported");
398
if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
399
log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
400
num_elem, type2name(elem_bt));
405
// When using mask, mask use type needs to be VecMaskUseLoad.
406
VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
407
: is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
408
if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
409
log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
410
n, sopc, num_elem, type2name(elem_bt),
411
is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
412
return false; // not supported
415
// Return true if current platform has implemented the masked operation with predicate feature.
416
bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
417
if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
418
log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
419
n, sopc, num_elem, type2name(elem_bt));
423
Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
426
opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
427
if (opd3 == nullptr) {
428
log_if_needed(" ** unbox failed v3=%s",
429
NodeClassNames[argument(7)->Opcode()]);
435
opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
436
if (opd2 == nullptr) {
437
log_if_needed(" ** unbox failed v2=%s",
438
NodeClassNames[argument(6)->Opcode()]);
444
opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
445
if (opd1 == nullptr) {
446
log_if_needed(" ** unbox failed v1=%s",
447
NodeClassNames[argument(5)->Opcode()]);
452
default: fatal("unsupported arity: %d", n);
455
Node* mask = nullptr;
457
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
458
assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
459
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
460
mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
461
if (mask == nullptr) {
462
log_if_needed(" ** unbox failed mask=%s",
463
NodeClassNames[argument(n + 5)->Opcode()]);
468
Node* operation = nullptr;
469
if (opc == Op_CallLeafVector) {
470
assert(UseVectorStubs, "sanity");
471
operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
472
if (operation == nullptr) {
473
log_if_needed(" ** svml call failed for %s_%s_%d",
474
(elem_bt == T_FLOAT)?"float":"double",
475
VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
476
num_elem * type2aelembytes(elem_bt));
480
const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
484
operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
488
operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
491
default: fatal("unsupported arity: %d", n);
495
if (is_masked_op && mask != nullptr) {
497
operation->add_req(mask);
498
operation->add_flag(Node::Flag_is_predicated_vector);
500
operation->add_flag(Node::Flag_is_predicated_using_blend);
501
operation = gvn().transform(operation);
502
operation = new VectorBlendNode(opd1, operation, mask);
505
operation = gvn().transform(operation);
507
// Wrap it up in VectorBox to keep object type information.
508
Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
510
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
514
// Following routine generates IR corresponding to AbstractShuffle::partiallyWrapIndex method,
515
// which partially wraps index by modulo VEC_LENGTH and generates a negative index value if original
516
// index is out of valid index range [0, VEC_LENGTH)
518
// wrapped_index = (VEC_LENGTH - 1) & index
519
// if (index u> VEC_LENGTH) {
520
// wrapped_index -= VEC_LENGTH;
522
// Note: Unsigned greater than comparison treat both <0 and >VEC_LENGTH indices as out-of-bound
524
Node* LibraryCallKit::partially_wrap_indexes(Node* index_vec, int num_elem, BasicType elem_bt) {
525
assert(elem_bt == T_BYTE, "Shuffles use byte array based backing storage.");
526
const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
527
const Type* type_bt = Type::get_const_basic_type(elem_bt);
529
Node* mod_mask = gvn().makecon(TypeInt::make(num_elem-1));
530
Node* bcast_mod_mask = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, type_bt));
532
BoolTest::mask pred = BoolTest::ugt;
533
ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(pred));
534
Node* lane_cnt = gvn().makecon(TypeInt::make(num_elem));
535
Node* bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
536
const TypeVect* vmask_type = TypeVect::makemask(type_bt, num_elem);
537
Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
539
// Make the indices greater than lane count as -ve values to match the java side implementation.
540
index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
541
Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
542
return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
545
// <Sh extends VectorShuffle<E>, E>
546
// Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
547
// int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
548
bool LibraryCallKit::inline_vector_shuffle_iota() {
549
const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
550
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
551
const TypeInt* start_val = gvn().type(argument(4))->isa_int();
552
const TypeInt* step_val = gvn().type(argument(5))->isa_int();
553
const TypeInt* wrap = gvn().type(argument(6))->isa_int();
555
if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
556
vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
557
wrap == nullptr || !wrap->is_con()) {
558
return false; // not enough info for intrinsification
561
if (!is_klass_initialized(shuffle_klass)) {
562
log_if_needed(" ** klass argument not initialized");
566
int do_wrap = wrap->get_con();
567
int num_elem = vlen->get_con();
568
BasicType elem_bt = T_BYTE;
570
bool effective_indices_in_range = false;
571
if (start_val->is_con() && step_val->is_con()) {
572
int effective_min_index = start_val->get_con();
573
int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
574
effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
577
if (!do_wrap && !effective_indices_in_range) {
578
// Disable instrinsification for unwrapped shuffle iota if start/step
579
// values are non-constant OR if intermediate result overflows byte value range.
583
if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed) ||
584
!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed) ||
585
!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
586
!arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
591
(!arch_supports_vector(Op_SubVB, num_elem, elem_bt, VecMaskNotUsed) ||
592
!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskNotUsed) ||
593
!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskNotUsed))) {
597
bool step_multiply = !step_val->is_con() || !is_power_of_2(step_val->get_con());
598
if ((step_multiply && !arch_supports_vector(Op_MulVB, num_elem, elem_bt, VecMaskNotUsed)) ||
599
(!step_multiply && !arch_supports_vector(Op_LShiftVB, num_elem, elem_bt, VecMaskNotUsed))) {
603
const Type * type_bt = Type::get_const_basic_type(elem_bt);
604
const TypeVect * vt = TypeVect::make(type_bt, num_elem);
606
Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
608
Node* start = argument(4);
609
Node* step = argument(5);
612
Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
613
res = gvn().transform(VectorNode::make(Op_MulVB, res, bcast_step, vt));
614
} else if (step_val->get_con() > 1) {
615
Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
616
Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
617
res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
620
if (!start_val->is_con() || start_val->get_con() != 0) {
621
Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
622
res = gvn().transform(VectorNode::make(Op_AddVB, res, bcast_start, vt));
625
Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
626
Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
629
// Wrap the indices greater than lane count.
630
res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
632
res = partially_wrap_indexes(res, num_elem, elem_bt);
635
ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
636
const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
638
// Wrap it up in VectorBox to keep object type information.
639
res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
641
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
646
// long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
647
// int length, M m, VectorMaskOp<M> defaultImpl)
648
bool LibraryCallKit::inline_vector_mask_operation() {
649
const TypeInt* oper = gvn().type(argument(0))->isa_int();
650
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
651
const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
652
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
653
Node* mask = argument(4);
655
if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
656
return false; // dead code
659
if (!is_klass_initialized(mask_klass)) {
660
log_if_needed(" ** klass argument not initialized");
664
int num_elem = vlen->get_con();
665
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
666
BasicType elem_bt = elem_type->basic_type();
668
int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
669
if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
670
log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
671
mopc, num_elem, type2name(elem_bt));
672
return false; // not supported
675
const Type* elem_ty = Type::get_const_basic_type(elem_bt);
676
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
677
const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
678
Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
679
if (mask_vec == nullptr) {
680
log_if_needed(" ** unbox failed mask=%s",
681
NodeClassNames[argument(4)->Opcode()]);
685
if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
686
mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
688
const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
689
Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
690
if (mopc != Op_VectorMaskToLong) {
691
maskoper = ConvI2L(maskoper);
693
set_result(maskoper);
695
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
701
// Sh extends VectorShuffle<E>,
703
// V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
704
// Class<? extends Sh> shuffleClass, Sh s, int length,
705
// ShuffleToVectorOperation<V, Sh, E> defaultImpl)
706
bool LibraryCallKit::inline_vector_shuffle_to_vector() {
707
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
708
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
709
const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
710
Node* shuffle = argument(3);
711
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
713
if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
714
return false; // dead code
716
if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
717
return false; // not enough info for intrinsification
719
if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
720
log_if_needed(" ** klass argument not initialized");
724
int num_elem = vlen->get_con();
725
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
726
BasicType elem_bt = elem_type->basic_type();
732
int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
733
// Make sure that cast is implemented to particular type/size combination.
734
if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
735
log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
736
cast_vopc, num_elem, type2name(elem_bt));
740
ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
741
const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
743
// Unbox shuffle with true flag to indicate its load shuffle to vector
744
// shuffle is a byte array
745
Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
747
// cast byte to target element type
748
shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
750
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
751
const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
754
Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
756
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
762
// S extends VectorSpecies<E>,
764
// M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
765
// long bits, int mode, S s,
766
// BroadcastOperation<M, E, S> defaultImpl)
767
bool LibraryCallKit::inline_vector_frombits_coerced() {
768
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
769
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
770
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
771
const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
772
// Mode argument determines the mode of operation it can take following values:-
773
// MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
774
// MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
775
const TypeInt* mode = gvn().type(argument(5))->isa_int();
777
if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
778
bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
779
!vlen->is_con() || !mode->is_con()) {
780
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
781
NodeClassNames[argument(0)->Opcode()],
782
NodeClassNames[argument(1)->Opcode()],
783
NodeClassNames[argument(2)->Opcode()],
784
NodeClassNames[argument(5)->Opcode()]);
785
return false; // not enough info for intrinsification
788
if (!is_klass_initialized(vector_klass)) {
789
log_if_needed(" ** klass argument not initialized");
792
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
793
if (!elem_type->is_primitive_type()) {
794
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
795
return false; // should be primitive type
797
BasicType elem_bt = elem_type->basic_type();
798
int num_elem = vlen->get_con();
799
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
800
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
802
bool is_mask = is_vector_mask(vbox_klass);
803
int bcast_mode = mode->get_con();
804
VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
805
int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
807
if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
808
log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
809
num_elem, type2name(elem_bt),
812
return false; // not supported
815
Node* broadcast = nullptr;
816
Node* bits = argument(3);
819
if (opc == Op_VectorLongToMask) {
820
const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
821
if (vt->isa_vectmask()) {
822
broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
824
const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
825
broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
826
broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
830
case T_BOOLEAN: // fall-through
831
case T_BYTE: // fall-through
832
case T_SHORT: // fall-through
833
case T_CHAR: // fall-through
835
elem = gvn().transform(new ConvL2INode(bits));
839
elem = gvn().transform(new MoveL2DNode(bits));
843
bits = gvn().transform(new ConvL2INode(bits));
844
elem = gvn().transform(new MoveI2FNode(bits));
848
// no conversion needed
851
default: fatal("%s", type2name(elem_bt));
853
broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
854
broadcast = gvn().transform(broadcast);
857
Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
859
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
863
static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
864
assert(arr_type != nullptr, "unexpected");
865
BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
866
if (elem_bt == arr_elem_bt) {
868
} else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
869
// Load/store of short vector from/to char[] is supported
871
} else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
872
// Load/store of byte vector from/to boolean[] is supported
875
return mismatched_ms;
881
// VM extends VectorPayload,
883
// S extends VectorSpecies<E>>
884
// VM load(Class<? extends VM> vmClass, Class<E> eClass,
886
// Object base, long offset, // Unsafe addressing
887
// boolean fromSegment,
888
// C container, long index, S s, // Arguments for default implementation
889
// LoadOperation<C, VM, S> defaultImpl) {
892
// V extends VectorPayload>
893
// void store(Class<?> vClass, Class<?> eClass,
895
// Object base, long offset, // Unsafe addressing
896
// boolean fromSegment,
897
// V v, C container, long index, // Arguments for default implementation
898
// StoreVectorOperation<C, V> defaultImpl) {
899
bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
900
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
901
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
902
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
903
const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
905
if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
906
vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
907
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
908
NodeClassNames[argument(0)->Opcode()],
909
NodeClassNames[argument(1)->Opcode()],
910
NodeClassNames[argument(2)->Opcode()],
911
NodeClassNames[argument(6)->Opcode()]);
912
return false; // not enough info for intrinsification
914
if (!is_klass_initialized(vector_klass)) {
915
log_if_needed(" ** klass argument not initialized");
919
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
920
if (!elem_type->is_primitive_type()) {
921
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
922
return false; // should be primitive type
924
BasicType elem_bt = elem_type->basic_type();
925
int num_elem = vlen->get_con();
927
// TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
928
if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
929
log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
930
is_store, is_store ? "store" : "load",
931
num_elem, type2name(elem_bt));
932
return false; // not supported
935
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
936
bool is_mask = is_vector_mask(vbox_klass);
938
Node* base = argument(3);
939
Node* offset = ConvL2X(argument(4));
941
// Save state and restore on bailout
943
SafePointNode* old_map = clone_map();
945
Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
947
// The memory barrier checks are based on ones for unsafe access.
948
// This is not 1-1 implementation.
949
const Type *const base_type = gvn().type(base);
951
const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
952
const TypeAryPtr* arr_type = addr_type->isa_aryptr();
954
const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
955
const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
957
const bool is_mixed_access = !in_heap && !in_native;
959
const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
961
const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
963
// For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
964
// with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
965
// loading from or before storing to backing storage which is mandatory for semantic correctness of
966
// big-endian memory layout.
967
bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
968
BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
969
arr_type->elem()->array_element_basic_type() != elem_bt);
970
BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
971
if (!is_java_primitive(mem_elem_bt)) {
972
log_if_needed(" ** non-primitive array element type");
975
int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
976
if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
977
log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
978
is_store, is_store ? "store" : "load",
979
num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
985
// In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
988
if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
989
|| !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
990
log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
992
num_elem, type2name(elem_bt));
995
return false; // not supported
998
if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
999
|| !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
1000
log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1002
mem_num_elem, type2name(mem_elem_bt));
1005
return false; // not supported
1011
if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1014
return false; // not supported
1017
if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1020
return false; // not supported
1025
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1027
if (needs_cpu_membar) {
1028
insert_mem_bar(Op_MemBarCPUOrder);
1032
Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1033
if (val == nullptr) {
1036
return false; // operand unboxing failed
1038
set_all_memory(reset_memory());
1040
// In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1041
int store_num_elem = num_elem;
1042
if (mismatched_ms) {
1043
store_num_elem = mem_num_elem;
1044
const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
1045
val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1048
val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1050
Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1051
set_memory(vstore, addr_type);
1053
// When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1054
Node* vload = nullptr;
1055
if (mismatched_ms) {
1056
vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1057
const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1058
vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1060
// Special handle for masks
1062
vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1063
vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1065
vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1068
Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1072
destruct_map_clone(old_map);
1074
if (needs_cpu_membar) {
1075
insert_mem_bar(Op_MemBarCPUOrder);
1078
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1084
// V extends Vector<?>,
1086
// S extends VectorSpecies<E>,
1087
// M extends VectorMask<E>>
1088
// V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1089
// int length, Object base, long offset, // Unsafe addressing
1090
// boolean fromSegment,
1091
// M m, int offsetInRange,
1092
// C container, long index, S s, // Arguments for default implementation
1093
// LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1096
// V extends Vector<E>,
1097
// M extends VectorMask<E>,
1099
// void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1101
// Object base, long offset, // Unsafe addressing
1102
// boolean fromSegment,
1103
// V v, M m, C container, long index, // Arguments for default implementation
1104
// StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1106
bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1107
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1108
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1109
const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1110
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1111
const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1113
if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1114
vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1115
elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
1116
log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1117
NodeClassNames[argument(0)->Opcode()],
1118
NodeClassNames[argument(1)->Opcode()],
1119
NodeClassNames[argument(2)->Opcode()],
1120
NodeClassNames[argument(3)->Opcode()],
1121
NodeClassNames[argument(7)->Opcode()]);
1122
return false; // not enough info for intrinsification
1124
if (!is_klass_initialized(vector_klass)) {
1125
log_if_needed(" ** klass argument not initialized");
1129
if (!is_klass_initialized(mask_klass)) {
1130
log_if_needed(" ** mask klass argument not initialized");
1134
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1135
if (!elem_type->is_primitive_type()) {
1136
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1137
return false; // should be primitive type
1140
BasicType elem_bt = elem_type->basic_type();
1141
int num_elem = vlen->get_con();
1143
Node* base = argument(4);
1144
Node* offset = ConvL2X(argument(5));
1146
// Save state and restore on bailout
1148
SafePointNode* old_map = clone_map();
1150
Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1151
const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1152
const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1154
bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1155
BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1156
// If there is no consistency between array and vector element types, it must be special byte array case
1157
if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1158
log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1159
is_store, is_store ? "storeMasked" : "loadMasked",
1160
num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1166
int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1167
BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1168
bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1169
mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1171
// If current arch does not support the predicated operations, we have to bail
1172
// out when current case uses the predicate feature.
1173
if (!supports_predicate) {
1174
bool needs_predicate = false;
1176
// Masked vector store always uses the predicated store.
1177
needs_predicate = true;
1179
// Masked vector load with IOOBE always uses the predicated load.
1180
const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1181
if (!offset_in_range->is_con()) {
1182
log_if_needed(" ** missing constant: offsetInRange=%s",
1183
NodeClassNames[argument(8)->Opcode()]);
1188
needs_predicate = (offset_in_range->get_con() == 0);
1191
if (needs_predicate) {
1192
log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1193
is_store ? "storeMasked" : "loadMasked",
1194
num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1201
// This only happens for masked vector load. If predicate is not supported, then check whether
1202
// the normal vector load and blend operations are supported by backend.
1203
if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1204
!arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1205
log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1206
num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1212
// Since we are using byte array, we need to double check that the vector reinterpret operation
1213
// with byte type is supported by backend.
1214
if (mismatched_ms) {
1215
if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1216
log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1217
is_store, is_store ? "storeMasked" : "loadMasked",
1218
num_elem, type2name(elem_bt));
1225
// Since it needs to unbox the mask, we need to double check that the related load operations
1226
// for mask are supported by backend.
1227
if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1228
log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1229
is_store, is_store ? "storeMasked" : "loadMasked",
1230
num_elem, type2name(elem_bt));
1236
// Can base be null? Otherwise, always on-heap access.
1237
bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1238
if (can_access_non_heap) {
1239
insert_mem_bar(Op_MemBarCPUOrder);
1242
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1243
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1244
assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1245
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1246
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1248
Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1249
if (mask == nullptr) {
1250
log_if_needed(" ** unbox failed mask=%s",
1251
is_store ? NodeClassNames[argument(9)->Opcode()]
1252
: NodeClassNames[argument(8)->Opcode()]);
1259
Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1260
if (val == nullptr) {
1261
log_if_needed(" ** unbox failed vector=%s",
1262
NodeClassNames[argument(8)->Opcode()]);
1265
return false; // operand unboxing failed
1267
set_all_memory(reset_memory());
1269
if (mismatched_ms) {
1270
// Reinterpret the incoming vector to byte vector.
1271
const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1272
val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1273
// Reinterpret the vector mask to byte type.
1274
const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1275
const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1276
mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1278
Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1279
set_memory(vstore, addr_type);
1281
Node* vload = nullptr;
1283
if (mismatched_ms) {
1284
// Reinterpret the vector mask to byte type.
1285
const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1286
const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1287
mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1290
if (supports_predicate) {
1291
// Generate masked load vector node if predicate feature is supported.
1292
const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1293
vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1295
// Use the vector blend to implement the masked load vector. The biased elements are zeros.
1296
Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1297
zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1298
vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1299
vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1302
if (mismatched_ms) {
1303
const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1304
vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1307
Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1311
destruct_map_clone(old_map);
1313
if (can_access_non_heap) {
1314
insert_mem_bar(Op_MemBarCPUOrder);
1317
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1322
// V extends Vector<?>,
1323
// W extends Vector<Integer>,
1324
// S extends VectorSpecies<E>,
1325
// M extends VectorMask<E>,
1327
// V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1328
// Class<? extends Vector<Integer>> vectorIndexClass,
1329
// Object base, long offset, // Unsafe addressing
1330
// W index_vector, M m,
1331
// C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1332
// LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1335
// V extends Vector<E>,
1336
// W extends Vector<Integer>,
1337
// M extends VectorMask<E>,
1339
// void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1340
// int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1341
// W index_vector, V v, M m,
1342
// C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1343
// StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1345
bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1346
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1347
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1348
const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1349
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1350
const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1352
if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1353
vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1354
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1355
NodeClassNames[argument(0)->Opcode()],
1356
NodeClassNames[argument(2)->Opcode()],
1357
NodeClassNames[argument(3)->Opcode()],
1358
NodeClassNames[argument(4)->Opcode()]);
1359
return false; // not enough info for intrinsification
1362
if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1363
log_if_needed(" ** klass argument not initialized");
1367
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1368
if (!elem_type->is_primitive_type()) {
1369
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1370
return false; // should be primitive type
1373
BasicType elem_bt = elem_type->basic_type();
1374
int num_elem = vlen->get_con();
1376
const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1377
bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1379
if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1380
log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1381
return false; // not enough info for intrinsification
1384
if (!is_klass_initialized(mask_klass)) {
1385
log_if_needed(" ** mask klass argument not initialized");
1389
if (vmask_type->maybe_null()) {
1390
log_if_needed(" ** null mask values are not allowed for masked op");
1394
// Check whether the predicated gather/scatter node is supported by architecture.
1395
VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1396
if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1397
log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1398
is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1399
num_elem, type2name(elem_bt));
1400
return false; // not supported
1403
// Check whether the normal gather/scatter node is supported for non-masked operation.
1404
if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1405
log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1406
is_scatter, is_scatter ? "scatter" : "gather",
1407
num_elem, type2name(elem_bt));
1408
return false; // not supported
1412
// Check that the vector holding indices is supported by architecture
1413
// For sub-word gathers expander receive index array.
1414
if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1415
log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1416
is_scatter, is_scatter ? "scatter" : "gather",
1417
num_elem, is_masked_op ? 1 : 0);
1418
return false; // not supported
1421
Node* base = argument(5);
1422
Node* offset = ConvL2X(argument(6));
1424
// Save state and restore on bailout
1426
SafePointNode* old_map = clone_map();
1428
Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1430
const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1431
const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1433
// The array must be consistent with vector type
1434
if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1435
log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1436
is_scatter, is_scatter ? "scatter" : "gather",
1437
num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1443
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1444
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1445
ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1446
if (vbox_idx_klass == nullptr) {
1452
Node* index_vect = nullptr;
1453
const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1454
if (!is_subword_type(elem_bt)) {
1455
index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1456
if (index_vect == nullptr) {
1463
Node* mask = nullptr;
1465
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1466
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1467
mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1468
if (mask == nullptr) {
1469
log_if_needed(" ** unbox failed mask=%s",
1470
is_scatter ? NodeClassNames[argument(10)->Opcode()]
1471
: NodeClassNames[argument(9)->Opcode()]);
1478
const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1480
Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1481
if (val == nullptr) {
1484
return false; // operand unboxing failed
1486
set_all_memory(reset_memory());
1488
Node* vstore = nullptr;
1489
if (mask != nullptr) {
1490
vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1492
vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1494
set_memory(vstore, addr_type);
1496
Node* vload = nullptr;
1497
Node* index = argument(11);
1498
Node* indexMap = argument(12);
1499
Node* indexM = argument(13);
1500
if (mask != nullptr) {
1501
if (is_subword_type(elem_bt)) {
1502
Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1503
vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1505
vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1508
if (is_subword_type(elem_bt)) {
1509
Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1510
vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1512
vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1515
Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1519
destruct_map_clone(old_map);
1521
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1526
// <V extends Vector<E>,
1527
// M extends VectorMask<E>,
1529
// long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1530
// Class<E> elementType, int length, V v, M m,
1531
// ReductionOperation<V, M> defaultImpl)
1532
bool LibraryCallKit::inline_vector_reduction() {
1533
const TypeInt* opr = gvn().type(argument(0))->isa_int();
1534
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1535
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1536
const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1537
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1539
if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1540
!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1541
log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1542
NodeClassNames[argument(0)->Opcode()],
1543
NodeClassNames[argument(1)->Opcode()],
1544
NodeClassNames[argument(3)->Opcode()],
1545
NodeClassNames[argument(4)->Opcode()]);
1546
return false; // not enough info for intrinsification
1548
if (!is_klass_initialized(vector_klass)) {
1549
log_if_needed(" ** klass argument not initialized");
1552
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1553
if (!elem_type->is_primitive_type()) {
1554
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1555
return false; // should be primitive type
1558
const Type* vmask_type = gvn().type(argument(6));
1559
bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1561
if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1562
log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1563
return false; // not enough info for intrinsification
1566
if (!is_klass_initialized(mask_klass)) {
1567
log_if_needed(" ** mask klass argument not initialized");
1571
if (vmask_type->maybe_null()) {
1572
log_if_needed(" ** null mask values are not allowed for masked op");
1577
BasicType elem_bt = elem_type->basic_type();
1578
int num_elem = vlen->get_con();
1579
int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1580
int sopc = ReductionNode::opcode(opc, elem_bt);
1582
// When using mask, mask use type needs to be VecMaskUseLoad.
1583
if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1584
log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1585
sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1589
// Return true if current platform has implemented the masked operation with predicate feature.
1590
bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1591
if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1592
log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1593
sopc, num_elem, type2name(elem_bt));
1597
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1598
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1600
Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1601
if (opd == nullptr) {
1602
return false; // operand unboxing failed
1605
Node* mask = nullptr;
1607
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1608
assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1609
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1610
mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1611
if (mask == nullptr) {
1612
log_if_needed(" ** unbox failed mask=%s",
1613
NodeClassNames[argument(6)->Opcode()]);
1618
Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1621
assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1622
if (mask != nullptr && !use_predicate) {
1623
Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1624
value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1627
// Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1628
// as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1629
value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1631
if (mask != nullptr && use_predicate) {
1632
value->add_req(mask);
1633
value->add_flag(Node::Flag_is_predicated_vector);
1636
value = gvn().transform(value);
1638
Node* bits = nullptr;
1643
bits = gvn().transform(new ConvI2LNode(value));
1647
value = gvn().transform(new MoveF2INode(value));
1648
bits = gvn().transform(new ConvI2LNode(value));
1652
bits = gvn().transform(new MoveD2LNode(value));
1656
bits = value; // no conversion needed
1659
default: fatal("%s", type2name(elem_bt));
1662
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1666
// public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1668
// BiFunction<V, V, Boolean> defaultImpl)
1670
bool LibraryCallKit::inline_vector_test() {
1671
const TypeInt* cond = gvn().type(argument(0))->isa_int();
1672
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1673
const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1674
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1676
if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1677
!cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1678
log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1679
NodeClassNames[argument(0)->Opcode()],
1680
NodeClassNames[argument(1)->Opcode()],
1681
NodeClassNames[argument(2)->Opcode()],
1682
NodeClassNames[argument(3)->Opcode()]);
1683
return false; // not enough info for intrinsification
1685
if (!is_klass_initialized(vector_klass)) {
1686
log_if_needed(" ** klass argument not initialized");
1689
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1690
if (!elem_type->is_primitive_type()) {
1691
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1692
return false; // should be primitive type
1694
BasicType elem_bt = elem_type->basic_type();
1695
int num_elem = vlen->get_con();
1696
BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1697
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1698
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1700
if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1701
log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1702
cond->get_con(), num_elem, type2name(elem_bt),
1703
is_vector_mask(vbox_klass));
1707
Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1709
if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1710
opd1->bottom_type()->isa_vectmask())) {
1711
opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1715
if (opd1 == nullptr || opd2 == nullptr) {
1716
return false; // operand unboxing failed
1719
Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1720
BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1721
opd1->bottom_type()->isa_vectmask(), num_elem);
1722
Node* bol = gvn().transform(new BoolNode(cmp, test));
1723
Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1726
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1731
// <V extends Vector<E>,
1732
// M extends VectorMask<E>,
1734
// V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1736
// VectorBlendOp<V, M, E> defaultImpl)
1737
bool LibraryCallKit::inline_vector_blend() {
1738
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1739
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1740
const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1741
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1743
if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1744
return false; // dead code
1746
if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1747
elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1748
log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1749
NodeClassNames[argument(0)->Opcode()],
1750
NodeClassNames[argument(1)->Opcode()],
1751
NodeClassNames[argument(2)->Opcode()],
1752
NodeClassNames[argument(3)->Opcode()]);
1753
return false; // not enough info for intrinsification
1755
if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1756
log_if_needed(" ** klass argument not initialized");
1759
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1760
if (!elem_type->is_primitive_type()) {
1761
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1762
return false; // should be primitive type
1764
BasicType elem_bt = elem_type->basic_type();
1765
BasicType mask_bt = elem_bt;
1766
int num_elem = vlen->get_con();
1768
if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1769
log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1770
num_elem, type2name(elem_bt));
1771
return false; // not supported
1773
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1774
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1776
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1777
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1779
Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1780
Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1781
Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1783
if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1784
return false; // operand unboxing failed
1787
Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1789
Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1791
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1796
// <V extends Vector<E>,
1797
// M extends VectorMask<E>,
1799
// M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1801
// VectorCompareOp<V,M> defaultImpl)
1802
bool LibraryCallKit::inline_vector_compare() {
1803
const TypeInt* cond = gvn().type(argument(0))->isa_int();
1804
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1805
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1806
const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1807
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1809
if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1810
return false; // dead code
1812
if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1813
elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1814
log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1815
NodeClassNames[argument(0)->Opcode()],
1816
NodeClassNames[argument(1)->Opcode()],
1817
NodeClassNames[argument(2)->Opcode()],
1818
NodeClassNames[argument(3)->Opcode()],
1819
NodeClassNames[argument(4)->Opcode()]);
1820
return false; // not enough info for intrinsification
1822
if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1823
log_if_needed(" ** klass argument not initialized");
1826
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1827
if (!elem_type->is_primitive_type()) {
1828
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1829
return false; // should be primitive type
1832
int num_elem = vlen->get_con();
1833
BasicType elem_bt = elem_type->basic_type();
1834
BasicType mask_bt = elem_bt;
1836
if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1837
if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1838
log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1839
cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1844
if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1845
log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1846
cond->get_con(), num_elem, type2name(elem_bt));
1850
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1851
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1853
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1854
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1856
Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1857
Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1859
bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1860
Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1861
if (is_masked_op && mask == nullptr) {
1862
log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1863
cond->get_con(), num_elem, type2name(elem_bt));
1867
bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1868
if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1869
log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1870
cond->get_con(), num_elem, type2name(elem_bt));
1874
if (v1 == nullptr || v2 == nullptr) {
1875
return false; // operand unboxing failed
1877
BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1878
ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1880
const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1881
Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1884
if (use_predicate) {
1885
operation->add_req(mask);
1886
operation->add_flag(Node::Flag_is_predicated_vector);
1888
operation = gvn().transform(operation);
1889
operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1893
operation = gvn().transform(operation);
1895
Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1897
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1902
// <V extends Vector<E>,
1903
// Sh extends VectorShuffle<E>,
1904
// M extends VectorMask<E>,
1906
// V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1908
// VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1909
bool LibraryCallKit::inline_vector_rearrange() {
1910
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1911
const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1912
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1913
const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1914
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1916
if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1917
return false; // dead code
1919
if (shuffle_klass->const_oop() == nullptr ||
1920
vector_klass->const_oop() == nullptr ||
1921
elem_klass->const_oop() == nullptr ||
1923
log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1924
NodeClassNames[argument(0)->Opcode()],
1925
NodeClassNames[argument(1)->Opcode()],
1926
NodeClassNames[argument(3)->Opcode()],
1927
NodeClassNames[argument(4)->Opcode()]);
1928
return false; // not enough info for intrinsification
1930
if (!is_klass_initialized(vector_klass) ||
1931
!is_klass_initialized(shuffle_klass)) {
1932
log_if_needed(" ** klass argument not initialized");
1935
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1936
if (!elem_type->is_primitive_type()) {
1937
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1938
return false; // should be primitive type
1940
BasicType elem_bt = elem_type->basic_type();
1941
BasicType shuffle_bt = elem_bt;
1942
int num_elem = vlen->get_con();
1944
if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1945
log_if_needed(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1946
num_elem, type2name(elem_bt));
1947
return false; // not supported
1950
bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1951
bool use_predicate = is_masked_op;
1953
(mask_klass == nullptr ||
1954
mask_klass->const_oop() == nullptr ||
1955
!is_klass_initialized(mask_klass))) {
1956
log_if_needed(" ** mask_klass argument not initialized");
1958
VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1959
if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1960
use_predicate = false;
1962
(!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1963
!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1964
!arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1965
log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1966
num_elem, type2name(elem_bt));
1967
return false; // not supported
1970
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1971
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1973
ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1974
const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1976
Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1977
Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1979
if (v1 == nullptr || shuffle == nullptr) {
1980
return false; // operand unboxing failed
1983
Node* mask = nullptr;
1985
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1986
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1987
mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1988
if (mask == nullptr) {
1989
log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1990
num_elem, type2name(elem_bt));
1995
Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1997
if (use_predicate) {
1998
rearrange->add_req(mask);
1999
rearrange->add_flag(Node::Flag_is_predicated_vector);
2001
const TypeVect* vt = v1->bottom_type()->is_vect();
2002
rearrange = gvn().transform(rearrange);
2003
Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2004
Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2005
rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2008
rearrange = gvn().transform(rearrange);
2010
Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2012
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2016
static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2017
address addr = nullptr;
2018
assert(UseVectorStubs, "sanity");
2019
assert(name_ptr != nullptr, "unexpected");
2020
assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2021
int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2024
case 64: //fallthough
2025
case 128: //fallthough
2026
case 256: //fallthough
2028
if (bt == T_FLOAT) {
2029
snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2030
addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2032
assert(bt == T_DOUBLE, "must be FP type only");
2033
snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2034
addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2038
snprintf(name_ptr, name_len, "invalid");
2047
Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2048
assert(UseVectorStubs, "sanity");
2049
assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2050
assert(opd1 != nullptr, "must not be null");
2051
const TypeVect* vt = TypeVect::make(bt, num_elem);
2052
const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2053
char name[100] = "";
2055
// Get address for svml method.
2056
address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2058
if (addr == nullptr) {
2062
assert(name[0] != '\0', "name must not be null");
2063
Node* operation = make_runtime_call(RC_VECTOR,
2070
return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2074
// <V extends Vector<E>,
2075
// M extends VectorMask<E>,
2077
// V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2078
// Class<E> elementType, int length,
2080
// VectorBroadcastIntOp<V, M> defaultImpl)
2081
bool LibraryCallKit::inline_vector_broadcast_int() {
2082
const TypeInt* opr = gvn().type(argument(0))->isa_int();
2083
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2084
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2085
const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2086
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2088
if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2089
return false; // dead code
2091
if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2092
log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2093
NodeClassNames[argument(0)->Opcode()],
2094
NodeClassNames[argument(1)->Opcode()],
2095
NodeClassNames[argument(3)->Opcode()],
2096
NodeClassNames[argument(4)->Opcode()]);
2097
return false; // not enough info for intrinsification
2099
if (!is_klass_initialized(vector_klass)) {
2100
log_if_needed(" ** klass argument not initialized");
2104
const Type* vmask_type = gvn().type(argument(7));
2105
bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2107
if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2108
log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2109
return false; // not enough info for intrinsification
2112
if (!is_klass_initialized(mask_klass)) {
2113
log_if_needed(" ** mask klass argument not initialized");
2117
if (vmask_type->maybe_null()) {
2118
log_if_needed(" ** null mask values are not allowed for masked op");
2123
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2124
if (!elem_type->is_primitive_type()) {
2125
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2126
return false; // should be primitive type
2129
int num_elem = vlen->get_con();
2130
BasicType elem_bt = elem_type->basic_type();
2131
int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2133
bool is_shift = VectorNode::is_shift_opcode(opc);
2134
bool is_rotate = VectorNode::is_rotate_opcode(opc);
2136
if (opc == 0 || (!is_shift && !is_rotate)) {
2137
log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2138
return false; // operation not supported
2141
int sopc = VectorNode::opcode(opc, elem_bt);
2143
log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2144
return false; // operation not supported
2147
Node* cnt = argument(6);
2148
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2149
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2150
const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2152
// If CPU supports vector constant rotate instructions pass it directly
2153
bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2154
Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2155
bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2157
VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2158
bool use_predicate = is_masked_op;
2160
if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2161
use_predicate = false;
2162
if (!is_masked_op ||
2163
(!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2164
!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2166
log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2167
sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2168
return false; // not supported
2172
Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2173
Node* opd2 = nullptr;
2175
opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2177
assert(is_rotate, "unexpected operation");
2178
if (!is_const_rotate) {
2179
const Type * type_bt = Type::get_const_basic_type(elem_bt);
2180
cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2181
opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2183
// Constant shift value.
2188
if (opd1 == nullptr || opd2 == nullptr) {
2192
Node* mask = nullptr;
2194
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2195
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2196
mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2197
if (mask == nullptr) {
2198
log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2203
Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2204
if (is_masked_op && mask != nullptr) {
2205
if (use_predicate) {
2206
operation->add_req(mask);
2207
operation->add_flag(Node::Flag_is_predicated_vector);
2209
operation = gvn().transform(operation);
2210
operation = new VectorBlendNode(opd1, operation, mask);
2213
operation = gvn().transform(operation);
2214
Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2216
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2220
// public static <VOUT extends VectorPayload,
2221
// VIN extends VectorPayload,
2222
// S extends VectorSpecies>
2223
// VOUT convert(int oprId,
2224
// Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2225
// Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2227
// VectorConvertOp<VOUT, VIN, S> defaultImpl)
2229
bool LibraryCallKit::inline_vector_convert() {
2230
const TypeInt* opr = gvn().type(argument(0))->isa_int();
2232
const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2233
const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2234
const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2236
const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2237
const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2238
const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2240
if (opr == nullptr ||
2241
vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2242
vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2243
return false; // dead code
2245
if (!opr->is_con() ||
2246
vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2247
vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2248
log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2249
NodeClassNames[argument(0)->Opcode()],
2250
NodeClassNames[argument(1)->Opcode()],
2251
NodeClassNames[argument(2)->Opcode()],
2252
NodeClassNames[argument(3)->Opcode()],
2253
NodeClassNames[argument(4)->Opcode()],
2254
NodeClassNames[argument(5)->Opcode()],
2255
NodeClassNames[argument(6)->Opcode()]);
2256
return false; // not enough info for intrinsification
2258
if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2259
log_if_needed(" ** klass argument not initialized");
2263
assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2264
opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2265
opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2266
bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2267
bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2269
ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2270
ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2271
if (is_vector_shuffle(vbox_klass_from)) {
2272
return false; // vector shuffles aren't supported
2274
bool is_mask = is_vector_mask(vbox_klass_from);
2276
ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2277
if (!elem_type_from->is_primitive_type()) {
2278
return false; // should be primitive type
2280
BasicType elem_bt_from = elem_type_from->basic_type();
2281
ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2282
if (!elem_type_to->is_primitive_type()) {
2283
return false; // should be primitive type
2285
BasicType elem_bt_to = elem_type_to->basic_type();
2287
int num_elem_from = vlen_from->get_con();
2288
int num_elem_to = vlen_to->get_con();
2290
// Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2291
// since we may need to change size.
2292
if (!arch_supports_vector(Op_VectorReinterpret,
2295
is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2296
log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2297
is_cast ? "cast" : "reinterpret",
2298
num_elem_from, type2name(elem_bt_from), is_mask);
2302
// Check whether we can support resizing/reinterpreting to the new size.
2303
if (!arch_supports_vector(Op_VectorReinterpret,
2306
is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2307
log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2308
is_cast ? "cast" : "reinterpret",
2309
num_elem_to, type2name(elem_bt_to), is_mask);
2314
if (is_vector_shuffle(vbox_klass_to) &&
2315
(!arch_supports_vector(Op_SubVB, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2316
!arch_supports_vector(Op_VectorBlend, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2317
!arch_supports_vector(Op_VectorMaskCmp, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2318
!arch_supports_vector(Op_AndV, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2319
!arch_supports_vector(Op_Replicate, num_elem_to, elem_bt_to, VecMaskNotUsed))) {
2320
log_if_needed(" ** not supported: arity=1 op=shuffle_index_wrap vlen2=%d etype2=%s",
2321
num_elem_to, type2name(elem_bt_to));
2325
// At this point, we know that both input and output vector registers are supported
2326
// by the architecture. Next check if the casted type is simply to same type - which means
2327
// that it is actually a resize and not a cast.
2328
if (is_cast && elem_bt_from == elem_bt_to) {
2332
const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2334
Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2335
if (opd1 == nullptr) {
2339
const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2340
const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2342
// Safety check to prevent casting if source mask is of type vector
2343
// and destination mask of type predicate vector and vice-versa.
2344
// From X86 standpoint, this case will only arise over KNL target,
2345
// where certain masks (depending on the species) are either propagated
2346
// through a vector or predicate register.
2348
((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2349
(dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2355
assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2356
int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2358
// Make sure that vector cast is implemented to particular type/size combination if it is
2359
// not a mask casting.
2360
if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2361
log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2362
cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2366
if (num_elem_from < num_elem_to) {
2367
// Since input and output number of elements are not consistent, we need to make sure we
2368
// properly size. Thus, first make a cast that retains the number of elements from source.
2369
int num_elem_for_cast = num_elem_from;
2371
// It is possible that arch does not support this intermediate vector size
2372
// TODO More complex logic required here to handle this corner case for the sizes.
2373
if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2374
log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2376
num_elem_for_cast, type2name(elem_bt_to), is_mask);
2380
op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2381
// Now ensure that the destination gets properly resized to needed size.
2382
op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2383
} else if (num_elem_from > num_elem_to) {
2384
// Since number of elements from input is larger than output, simply reduce size of input
2385
// (we are supposed to drop top elements anyway).
2386
int num_elem_for_resize = num_elem_to;
2388
// It is possible that arch does not support this intermediate vector size
2389
// TODO More complex logic required here to handle this corner case for the sizes.
2390
if (!arch_supports_vector(Op_VectorReinterpret,
2391
num_elem_for_resize,
2394
log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2395
num_elem_for_resize, type2name(elem_bt_from), is_mask);
2399
const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2400
op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2401
op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2402
} else { // num_elem_from == num_elem_to
2404
// Make sure that cast for vector mask is implemented to particular type/size combination.
2405
if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2406
log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2407
num_elem_to, type2name(elem_bt_to), is_mask);
2410
op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2412
// Since input and output number of elements match, and since we know this vector size is
2413
// supported, simply do a cast with no resize needed.
2414
op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2417
} else if (!Type::equals(src_type, dst_type)) {
2418
assert(!is_cast, "must be reinterpret");
2419
op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2422
if (is_vector_shuffle(vbox_klass_to)) {
2423
op = partially_wrap_indexes(op, num_elem_to, elem_bt_to);
2426
const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2427
Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2429
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2434
// <V extends Vector<E>,
2436
// V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2437
// V vec, int ix, long val,
2438
// VecInsertOp<V> defaultImpl)
2439
bool LibraryCallKit::inline_vector_insert() {
2440
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2441
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2442
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2443
const TypeInt* idx = gvn().type(argument(4))->isa_int();
2445
if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2446
return false; // dead code
2448
if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2449
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2450
NodeClassNames[argument(0)->Opcode()],
2451
NodeClassNames[argument(1)->Opcode()],
2452
NodeClassNames[argument(2)->Opcode()],
2453
NodeClassNames[argument(4)->Opcode()]);
2454
return false; // not enough info for intrinsification
2456
if (!is_klass_initialized(vector_klass)) {
2457
log_if_needed(" ** klass argument not initialized");
2460
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2461
if (!elem_type->is_primitive_type()) {
2462
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2463
return false; // should be primitive type
2465
BasicType elem_bt = elem_type->basic_type();
2466
int num_elem = vlen->get_con();
2467
if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2468
log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2469
num_elem, type2name(elem_bt));
2470
return false; // not supported
2473
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2474
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2476
Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2477
if (opd == nullptr) {
2481
Node* insert_val = argument(5);
2482
assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2484
// Convert insert value back to its appropriate type.
2487
insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2490
insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2493
insert_val = gvn().transform(new ConvL2INode(insert_val));
2496
insert_val = gvn().transform(new ConvL2INode(insert_val));
2497
insert_val = gvn().transform(new MoveI2FNode(insert_val));
2500
insert_val = gvn().transform(new MoveL2DNode(insert_val));
2503
// no conversion needed
2505
default: fatal("%s", type2name(elem_bt)); break;
2508
Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2510
Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2512
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2517
// <VM extends VectorPayload,
2519
// long extract(Class<? extends VM> vClass, Class<E> eClass,
2522
// VecExtractOp<VM> defaultImpl)
2523
bool LibraryCallKit::inline_vector_extract() {
2524
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2525
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2526
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2527
const TypeInt* idx = gvn().type(argument(4))->isa_int();
2529
if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2530
return false; // dead code
2532
if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2533
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2534
NodeClassNames[argument(0)->Opcode()],
2535
NodeClassNames[argument(1)->Opcode()],
2536
NodeClassNames[argument(2)->Opcode()]);
2537
return false; // not enough info for intrinsification
2539
if (!is_klass_initialized(vector_klass)) {
2540
log_if_needed(" ** klass argument not initialized");
2543
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2544
if (!elem_type->is_primitive_type()) {
2545
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2546
return false; // should be primitive type
2548
BasicType elem_bt = elem_type->basic_type();
2549
int num_elem = vlen->get_con();
2551
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2552
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2554
Node* opd = nullptr;
2556
if (is_vector_mask(vbox_klass)) {
2557
// vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2559
Node* pos = argument(4); // can be variable
2560
if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2561
// Transform mask to vector with type of boolean and utilize ExtractUB node.
2562
opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2563
if (opd == nullptr) {
2566
opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2567
opd = gvn().transform(new ExtractUBNode(opd, pos));
2568
opd = gvn().transform(new ConvI2LNode(opd));
2569
} else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2570
opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2571
if (opd == nullptr) {
2574
// VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2575
if (opd->bottom_type()->isa_vectmask() == nullptr) {
2576
opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2578
// ((toLong() >>> pos) & 1L
2579
opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2580
opd = gvn().transform(new URShiftLNode(opd, pos));
2581
opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2583
log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2584
return false; // not supported
2587
// vbox_klass is vector. This is used for Vector.lane(int).
2588
if (!idx->is_con()) {
2589
log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2590
return false; // not enough info for intrinsification
2593
int vopc = ExtractNode::opcode(elem_bt);
2594
if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2595
log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2596
num_elem, type2name(elem_bt));
2597
return false; // not supported
2600
opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2601
if (opd == nullptr) {
2604
ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2606
opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2611
opd = gvn().transform(new ConvI2LNode(opd));
2615
opd = gvn().transform(new MoveF2INode(opd));
2616
opd = gvn().transform(new ConvI2LNode(opd));
2620
opd = gvn().transform(new MoveD2LNode(opd));
2624
// no conversion needed
2627
default: fatal("%s", type2name(elem_bt));
2635
// <V extends Vector<E>,
2636
// M extends VectorMask<E>,
2638
// V compressExpandOp(int opr,
2639
// Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2640
// int length, V v, M m,
2641
// CompressExpandOperation<V, M> defaultImpl)
2642
bool LibraryCallKit::inline_vector_compress_expand() {
2643
const TypeInt* opr = gvn().type(argument(0))->isa_int();
2644
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2645
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2646
const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2647
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2649
if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2650
vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2651
elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2652
log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2653
NodeClassNames[argument(0)->Opcode()],
2654
NodeClassNames[argument(1)->Opcode()],
2655
NodeClassNames[argument(2)->Opcode()],
2656
NodeClassNames[argument(3)->Opcode()],
2657
NodeClassNames[argument(4)->Opcode()]);
2658
return false; // not enough info for intrinsification
2661
if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2662
log_if_needed(" ** klass argument not initialized");
2666
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2667
if (!elem_type->is_primitive_type()) {
2668
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2669
return false; // should be primitive type
2672
int num_elem = vlen->get_con();
2673
BasicType elem_bt = elem_type->basic_type();
2674
int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2676
if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2677
log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2678
opc, num_elem, type2name(elem_bt));
2679
return false; // not supported
2682
Node* opd1 = nullptr;
2683
const TypeInstPtr* vbox_type = nullptr;
2684
if (opc != Op_CompressM) {
2685
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2686
vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2687
opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2688
if (opd1 == nullptr) {
2689
log_if_needed(" ** unbox failed vector=%s",
2690
NodeClassNames[argument(5)->Opcode()]);
2695
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2696
assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2697
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2699
Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2700
if (mask == nullptr) {
2701
log_if_needed(" ** unbox failed mask=%s",
2702
NodeClassNames[argument(6)->Opcode()]);
2706
const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2707
Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2709
// Wrap it up in VectorBox to keep object type information.
2710
const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2711
Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2713
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2718
// <V extends Vector<E>,
2720
// S extends VectorSpecies<E>>
2721
// V indexVector(Class<? extends V> vClass, Class<E> eClass,
2723
// V v, int step, S s,
2724
// IndexOperation<V, S> defaultImpl)
2725
bool LibraryCallKit::inline_index_vector() {
2726
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2727
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2728
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2730
if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2731
vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2732
elem_klass->const_oop() == nullptr) {
2733
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2734
NodeClassNames[argument(0)->Opcode()],
2735
NodeClassNames[argument(1)->Opcode()],
2736
NodeClassNames[argument(2)->Opcode()]);
2737
return false; // not enough info for intrinsification
2740
if (!is_klass_initialized(vector_klass)) {
2741
log_if_needed(" ** klass argument not initialized");
2745
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2746
if (!elem_type->is_primitive_type()) {
2747
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2748
return false; // should be primitive type
2751
int num_elem = vlen->get_con();
2752
BasicType elem_bt = elem_type->basic_type();
2754
// Check whether the iota index generation op is supported by the current hardware
2755
if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2756
log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2757
return false; // not supported
2760
int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2761
int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2762
bool needs_mul = true;
2763
Node* scale = argument(4);
2764
const TypeInt* scale_type = gvn().type(scale)->isa_int();
2765
// Multiply is not needed if the scale is a constant "1".
2766
if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2769
// Check whether the vector multiply op is supported by the current hardware
2770
if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2771
log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2772
return false; // not supported
2775
// Check whether the scalar cast op is supported by the current hardware
2776
if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2777
int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2778
elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2779
if (!Matcher::match_rule_supported(cast_op)) {
2780
log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2781
NodeClassNames[cast_op]);
2782
return false; // not supported
2787
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2788
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2789
Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2790
if (opd == nullptr) {
2791
log_if_needed(" ** unbox failed vector=%s",
2792
NodeClassNames[argument(3)->Opcode()]);
2796
int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2797
int vadd_op = VectorNode::opcode(add_op, elem_bt);
2798
bool needs_add = true;
2799
// The addition is not needed if all the element values of "opd" are zero
2800
if (VectorNode::is_all_zeros_vector(opd)) {
2803
// Check whether the vector addition op is supported by the current hardware
2804
if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2805
log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2806
return false; // not supported
2810
// Compute the iota indice vector
2811
const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2812
Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2814
// Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2817
case T_BOOLEAN: // fall-through
2818
case T_BYTE: // fall-through
2819
case T_SHORT: // fall-through
2820
case T_CHAR: // fall-through
2822
// no conversion needed
2826
scale = gvn().transform(new ConvI2LNode(scale));
2830
scale = gvn().transform(new ConvI2FNode(scale));
2834
scale = gvn().transform(new ConvI2DNode(scale));
2837
default: fatal("%s", type2name(elem_bt));
2839
scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, Type::get_const_basic_type(elem_bt)));
2840
index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
2843
// Add "opd" if addition is needed.
2845
index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
2847
Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
2849
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2855
// M extends VectorMask<E>>
2856
// M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
2857
// long offset, long limit,
2858
// IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
2859
bool LibraryCallKit::inline_index_partially_in_upper_range() {
2860
const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
2861
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2862
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2864
if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2865
mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2866
log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
2867
NodeClassNames[argument(0)->Opcode()],
2868
NodeClassNames[argument(1)->Opcode()],
2869
NodeClassNames[argument(2)->Opcode()]);
2870
return false; // not enough info for intrinsification
2873
if (!is_klass_initialized(mask_klass)) {
2874
log_if_needed(" ** klass argument not initialized");
2878
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2879
if (!elem_type->is_primitive_type()) {
2880
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2881
return false; // should be primitive type
2884
int num_elem = vlen->get_con();
2885
BasicType elem_bt = elem_type->basic_type();
2887
// Check whether the necessary ops are supported by current hardware.
2888
bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
2889
if (!supports_mask_gen) {
2890
if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
2891
!arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
2892
!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
2893
log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2894
return false; // not supported
2897
// Check whether the scalar cast operation is supported by current hardware.
2898
if (elem_bt != T_LONG) {
2899
int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
2900
: (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
2901
if (!Matcher::match_rule_supported(cast_op)) {
2902
log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2903
NodeClassNames[cast_op]);
2904
return false; // not supported
2909
Node* offset = argument(3);
2910
Node* limit = argument(5);
2911
if (offset == nullptr || limit == nullptr) {
2912
log_if_needed(" ** offset or limit argument is null");
2913
return false; // not supported
2916
ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2917
assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
2918
const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
2920
// We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
2921
// So directly get indexLimit with "indexLimit = limit - offset".
2922
Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
2923
Node* mask = nullptr;
2924
if (supports_mask_gen) {
2925
mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
2927
// Generate the vector mask based on "mask = iota < indexLimit".
2928
// Broadcast "indexLimit" to a vector.
2930
case T_BOOLEAN: // fall-through
2931
case T_BYTE: // fall-through
2932
case T_SHORT: // fall-through
2933
case T_CHAR: // fall-through
2935
indexLimit = gvn().transform(new ConvL2INode(indexLimit));
2939
indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
2943
indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
2947
// no conversion needed
2950
default: fatal("%s", type2name(elem_bt));
2952
indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, Type::get_const_basic_type(elem_bt)));
2954
// Load the "iota" vector.
2955
const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2956
Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2958
// Compute the vector mask with "mask = iota < indexLimit".
2959
ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
2960
const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
2961
mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
2963
Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
2965
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2969
#undef non_product_log_if_needed