jdk

1
/*
2
 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4
 *
5
 * This code is free software; you can redistribute it and/or modify it
6
 * under the terms of the GNU General Public License version 2 only, as
7
 * published by the Free Software Foundation.
8
 *
9
 * This code is distributed in the hope that it will be useful, but WITHOUT
10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12
 * version 2 for more details (a copy is included in the LICENSE file that
13
 * accompanied this code).
14
 *
15
 * You should have received a copy of the GNU General Public License version
16
 * 2 along with this work; if not, write to the Free Software Foundation,
17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18
 *
19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20
 * or visit www.oracle.com if you need additional information or have any
21
 * questions.
22
 *
23
 */
24

25
#include "precompiled.hpp"
26
#include "ci/ciSymbols.hpp"
27
#include "classfile/vmSymbols.hpp"
28
#include "opto/library_call.hpp"
29
#include "opto/runtime.hpp"
30
#include "opto/vectornode.hpp"
31
#include "prims/vectorSupport.hpp"
32
#include "runtime/stubRoutines.hpp"
33

34
#ifdef ASSERT
35
static bool is_vector(ciKlass* klass) {
36
  return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37
}
38

39
static bool check_vbox(const TypeInstPtr* vbox_type) {
40
  assert(vbox_type->klass_is_exact(), "");
41

42
  ciInstanceKlass* ik = vbox_type->instance_klass();
43
  assert(is_vector(ik), "not a vector");
44

45
  ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46
  assert(fd1 != nullptr, "element type info is missing");
47

48
  ciConstant val1 = fd1->constant_value();
49
  BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50
  assert(is_java_primitive(elem_bt), "element type info is missing");
51

52
  ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53
  assert(fd2 != nullptr, "vector length info is missing");
54

55
  ciConstant val2 = fd2->constant_value();
56
  assert(val2.as_int() > 0, "vector length info is missing");
57

58
  return true;
59
}
60
#endif
61

62
#define log_if_needed(...)        \
63
  if (C->print_intrinsics()) {    \
64
    tty->print_cr(__VA_ARGS__);   \
65
  }
66

67
#ifndef PRODUCT
68
#define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69
#else
70
#define non_product_log_if_needed(...)
71
#endif
72

73
static bool is_vector_mask(ciKlass* klass) {
74
  return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75
}
76

77
static bool is_vector_shuffle(ciKlass* klass) {
78
  return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
79
}
80

81
bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
82
                                                 VectorMaskUseType mask_use_type, bool has_scalar_args) {
83
  bool is_supported = true;
84

85
  // has_scalar_args flag is true only for non-constant scalar shift count,
86
  // since in this case shift needs to be broadcasted.
87
  if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
88
       (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
89
    is_supported = false;
90
  }
91

92
  if (is_supported) {
93
    // Check if mask unboxing is supported, this is a two step process which first loads the contents
94
    // of boolean array into vector followed by either lane expansion to match the lane size of masked
95
    // vector operation or populate the predicate register.
96
    if ((mask_use_type & VecMaskUseLoad) != 0) {
97
      if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
98
          !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
99
        non_product_log_if_needed("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
100
                                  NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
101
        return false;
102
      }
103
    }
104

105
    if ((mask_use_type & VecMaskUsePred) != 0) {
106
      if (!Matcher::has_predicated_vectors() ||
107
          !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
108
        non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
109
                                  NodeClassNames[opc], type2name(elem_bt), num_elem);
110
        return false;
111
      }
112
    }
113
  }
114

115
  int lshiftopc, rshiftopc;
116
  switch(elem_bt) {
117
    case T_BYTE:
118
      lshiftopc = Op_LShiftI;
119
      rshiftopc = Op_URShiftB;
120
      break;
121
    case T_SHORT:
122
      lshiftopc = Op_LShiftI;
123
      rshiftopc = Op_URShiftS;
124
      break;
125
    case T_INT:
126
      lshiftopc = Op_LShiftI;
127
      rshiftopc = Op_URShiftI;
128
      break;
129
    case T_LONG:
130
      lshiftopc = Op_LShiftL;
131
      rshiftopc = Op_URShiftL;
132
      break;
133
    default: fatal("Unexpected type: %s", type2name(elem_bt));
134
  }
135
  int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
136
  int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
137
  if (!is_supported &&
138
      arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
139
      arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
140
      arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
141
    is_supported = true;
142
  }
143
  return is_supported;
144
}
145

146
Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
147
  assert(EnableVectorSupport, "");
148

149
  PreserveReexecuteState preexecs(this);
150
  jvms()->set_should_reexecute(true);
151

152
  VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
153
  set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
154
  make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
155
  set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
156
  set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
157
  Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
158

159
  assert(check_vbox(vbox_type), "");
160
  const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
161
  VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
162
  return gvn().transform(vbox);
163
}
164

165
Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
166
  assert(EnableVectorSupport, "");
167
  const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
168
  if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
169
    return nullptr; // arguments don't agree on vector shapes
170
  }
171
  if (vbox_type_v->maybe_null()) {
172
    return nullptr; // no nulls are allowed
173
  }
174
  assert(check_vbox(vbox_type), "");
175
  const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
176
  Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
177
  return unbox;
178
}
179

180
Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
181
  assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
182
  juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
183
  Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
184
  Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
185
  return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
186
}
187

188
bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
189
  // Check that the operation is valid.
190
  if (sopc <= 0) {
191
    non_product_log_if_needed("  ** Rejected intrinsification because no valid vector op could be extracted");
192
    return false;
193
  }
194

195
  if (VectorNode::is_vector_rotate(sopc)) {
196
    if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
197
      non_product_log_if_needed("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
198
                                NodeClassNames[sopc], type2name(type), num_elem);
199
      return false;
200
    }
201
  } else if (VectorNode::is_vector_integral_negate(sopc)) {
202
    if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
203
      non_product_log_if_needed("  ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
204
                                NodeClassNames[sopc], type2name(type), num_elem);
205
      return false;
206
    }
207
  } else {
208
    // Check that architecture supports this op-size-type combination.
209
    if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
210
      non_product_log_if_needed("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
211
                                NodeClassNames[sopc], type2name(type), num_elem);
212
      return false;
213
    } else {
214
      assert(Matcher::match_rule_supported(sopc), "must be supported");
215
    }
216
  }
217

218
  if (num_elem == 1) {
219
    if (mask_use_type != VecMaskNotUsed) {
220
      non_product_log_if_needed("  ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
221
                                NodeClassNames[sopc], type2name(type), num_elem);
222
      return false;
223
    }
224

225
    if (sopc != 0) {
226
      if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
227
        non_product_log_if_needed("  ** Not a svml call or load/store vector op (%s,%s,%d)",
228
                                  NodeClassNames[sopc], type2name(type), num_elem);
229
        return false;
230
      }
231
    }
232
  }
233

234
  if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
235
      Matcher::supports_vector_variable_shifts() == false) {
236
    log_if_needed("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
237
                  NodeClassNames[sopc], type2name(type), num_elem);
238
    return false;
239
  }
240

241
  // Check if mask unboxing is supported, this is a two step process which first loads the contents
242
  // of boolean array into vector followed by either lane expansion to match the lane size of masked
243
  // vector operation or populate the predicate register.
244
  if ((mask_use_type & VecMaskUseLoad) != 0) {
245
    if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
246
        !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
247
      non_product_log_if_needed("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
248
                                NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
249
      return false;
250
    }
251
  }
252

253
  // Check if mask boxing is supported, this is a two step process which first stores the contents
254
  // of mask vector / predicate register into a boolean vector followed by vector store operation to
255
  // transfer the contents to underlined storage of mask boxes which is a boolean array.
256
  if ((mask_use_type & VecMaskUseStore) != 0) {
257
    if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
258
        !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
259
      non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
260
                                NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
261
      return false;
262
    }
263
  }
264

265
  if ((mask_use_type & VecMaskUsePred) != 0) {
266
    bool is_supported = false;
267
    if (Matcher::has_predicated_vectors()) {
268
      if (VectorNode::is_vector_integral_negate(sopc)) {
269
        is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
270
      } else {
271
        is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
272
      }
273
    }
274
    is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
275

276
    if (!is_supported) {
277
      non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
278
                                NodeClassNames[sopc], type2name(type), num_elem);
279
      return false;
280
    }
281
  }
282

283
  return true;
284
}
285

286
static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
287
  if (vec_klass->const_oop() == nullptr) {
288
    return false; // uninitialized or some kind of unsafe access
289
  }
290
  assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
291
  ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
292
  return klass->is_initialized();
293
}
294

295
// public static
296
// <V extends Vector<E>,
297
//  M extends VectorMask<E>,
298
//  E>
299
// V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
300
//           int length, V v, M m,
301
//           UnaryOperation<V, M> defaultImpl)
302
//
303
// public static
304
// <V,
305
//  M extends VectorMask<E>,
306
//  E>
307
// V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308
//            int length, V v1, V v2, M m,
309
//            BinaryOperation<V, M> defaultImpl)
310
//
311
// public static
312
// <V extends Vector<E>,
313
//  M extends VectorMask<E>,
314
//  E>
315
// V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316
//             int length, V v1, V v2, V v3, M m,
317
//             TernaryOperation<V, M> defaultImpl)
318
//
319
bool LibraryCallKit::inline_vector_nary_operation(int n) {
320
  const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
321
  const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322
  const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
323
  const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
324
  const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
325

326
  if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
327
      !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
328
    log_if_needed("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
329
                    NodeClassNames[argument(0)->Opcode()],
330
                    NodeClassNames[argument(1)->Opcode()],
331
                    NodeClassNames[argument(3)->Opcode()],
332
                    NodeClassNames[argument(4)->Opcode()]);
333
    return false; // not enough info for intrinsification
334
  }
335

336
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
337
  if (!elem_type->is_primitive_type()) {
338
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
339
    return false; // should be primitive type
340
  }
341
  if (!is_klass_initialized(vector_klass)) {
342
    log_if_needed("  ** klass argument not initialized");
343
    return false;
344
  }
345

346
  // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
347
  // is used to control this operation.
348
  const Type* vmask_type = gvn().type(argument(n + 5));
349
  bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
350
  if (is_masked_op) {
351
    if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
352
      log_if_needed("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
353
      return false; // not enough info for intrinsification
354
    }
355

356
    if (!is_klass_initialized(mask_klass)) {
357
      log_if_needed("  ** mask klass argument not initialized");
358
      return false;
359
    }
360

361
    if (vmask_type->maybe_null()) {
362
      log_if_needed("  ** null mask values are not allowed for masked op");
363
      return false;
364
    }
365
  }
366

367
  BasicType elem_bt = elem_type->basic_type();
368
  int num_elem = vlen->get_con();
369
  int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
370
  int sopc = VectorNode::opcode(opc, elem_bt);
371
  if ((opc != Op_CallLeafVector) && (sopc == 0)) {
372
    log_if_needed("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
373
    return false; // operation not supported
374
  }
375
  if (num_elem == 1) {
376
    if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
377
      log_if_needed("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
378
                      n, opc, num_elem, type2name(elem_bt));
379
      return false;
380
    }
381
  }
382
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
383
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
384

385
  if (is_vector_mask(vbox_klass)) {
386
    assert(!is_masked_op, "mask operations do not need mask to control");
387
  }
388

389
  if (opc == Op_CallLeafVector) {
390
    if (!UseVectorStubs) {
391
      log_if_needed("  ** vector stubs support is disabled");
392
      return false;
393
    }
394
    if (!Matcher::supports_vector_calling_convention()) {
395
      log_if_needed("  ** no vector calling conventions supported");
396
      return false;
397
    }
398
    if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
399
      log_if_needed("  ** vector size (vlen=%d, etype=%s) is not supported",
400
                      num_elem, type2name(elem_bt));
401
      return false;
402
    }
403
  }
404

405
  // When using mask, mask use type needs to be VecMaskUseLoad.
406
  VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
407
                                      : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
408
  if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
409
    log_if_needed("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
410
                    n, sopc, num_elem, type2name(elem_bt),
411
                    is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
412
    return false; // not supported
413
  }
414

415
  // Return true if current platform has implemented the masked operation with predicate feature.
416
  bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
417
  if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
418
    log_if_needed("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
419
                    n, sopc, num_elem, type2name(elem_bt));
420
    return false;
421
  }
422

423
  Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
424
  switch (n) {
425
    case 3: {
426
      opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
427
      if (opd3 == nullptr) {
428
        log_if_needed("  ** unbox failed v3=%s",
429
                        NodeClassNames[argument(7)->Opcode()]);
430
        return false;
431
      }
432
      // fall-through
433
    }
434
    case 2: {
435
      opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
436
      if (opd2 == nullptr) {
437
        log_if_needed("  ** unbox failed v2=%s",
438
                        NodeClassNames[argument(6)->Opcode()]);
439
        return false;
440
      }
441
      // fall-through
442
    }
443
    case 1: {
444
      opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
445
      if (opd1 == nullptr) {
446
        log_if_needed("  ** unbox failed v1=%s",
447
                        NodeClassNames[argument(5)->Opcode()]);
448
        return false;
449
      }
450
      break;
451
    }
452
    default: fatal("unsupported arity: %d", n);
453
  }
454

455
  Node* mask = nullptr;
456
  if (is_masked_op) {
457
    ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
458
    assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
459
    const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
460
    mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
461
    if (mask == nullptr) {
462
      log_if_needed("  ** unbox failed mask=%s",
463
                      NodeClassNames[argument(n + 5)->Opcode()]);
464
      return false;
465
    }
466
  }
467

468
  Node* operation = nullptr;
469
  if (opc == Op_CallLeafVector) {
470
    assert(UseVectorStubs, "sanity");
471
    operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
472
    if (operation == nullptr) {
473
      log_if_needed("  ** svml call failed for %s_%s_%d",
474
                         (elem_bt == T_FLOAT)?"float":"double",
475
                         VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
476
                         num_elem * type2aelembytes(elem_bt));
477
      return false;
478
     }
479
  } else {
480
    const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
481
    switch (n) {
482
      case 1:
483
      case 2: {
484
        operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
485
        break;
486
      }
487
      case 3: {
488
        operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
489
        break;
490
      }
491
      default: fatal("unsupported arity: %d", n);
492
    }
493
  }
494

495
  if (is_masked_op && mask != nullptr) {
496
    if (use_predicate) {
497
      operation->add_req(mask);
498
      operation->add_flag(Node::Flag_is_predicated_vector);
499
    } else {
500
      operation->add_flag(Node::Flag_is_predicated_using_blend);
501
      operation = gvn().transform(operation);
502
      operation = new VectorBlendNode(opd1, operation, mask);
503
    }
504
  }
505
  operation = gvn().transform(operation);
506

507
  // Wrap it up in VectorBox to keep object type information.
508
  Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
509
  set_result(vbox);
510
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
511
  return true;
512
}
513

514
// Following routine generates IR corresponding to AbstractShuffle::partiallyWrapIndex method,
515
// which partially wraps index by modulo VEC_LENGTH and generates a negative index value if original
516
// index is out of valid index range [0, VEC_LENGTH)
517
//
518
//   wrapped_index = (VEC_LENGTH - 1) & index
519
//   if (index u> VEC_LENGTH) {
520
//     wrapped_index -= VEC_LENGTH;
521
//
522
// Note: Unsigned greater than comparison treat both <0 and >VEC_LENGTH indices as out-of-bound
523
// indexes.
524
Node* LibraryCallKit::partially_wrap_indexes(Node* index_vec, int num_elem, BasicType elem_bt) {
525
  assert(elem_bt == T_BYTE, "Shuffles use byte array based backing storage.");
526
  const TypeVect* vt  = TypeVect::make(elem_bt, num_elem);
527
  const Type* type_bt = Type::get_const_basic_type(elem_bt);
528

529
  Node* mod_mask = gvn().makecon(TypeInt::make(num_elem-1));
530
  Node* bcast_mod_mask  = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, type_bt));
531

532
  BoolTest::mask pred = BoolTest::ugt;
533
  ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(pred));
534
  Node* lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
535
  Node* bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
536
  const TypeVect* vmask_type = TypeVect::makemask(type_bt, num_elem);
537
  Node*  mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
538

539
  // Make the indices greater than lane count as -ve values to match the java side implementation.
540
  index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
541
  Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
542
  return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
543
}
544

545
// <Sh extends VectorShuffle<E>,  E>
546
//  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
547
//                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
548
bool LibraryCallKit::inline_vector_shuffle_iota() {
549
  const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
550
  const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
551
  const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
552
  const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
553
  const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
554

555
  if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
556
      vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
557
      wrap == nullptr || !wrap->is_con()) {
558
    return false; // not enough info for intrinsification
559
  }
560

561
  if (!is_klass_initialized(shuffle_klass)) {
562
    log_if_needed("  ** klass argument not initialized");
563
    return false;
564
  }
565

566
  int do_wrap = wrap->get_con();
567
  int num_elem = vlen->get_con();
568
  BasicType elem_bt = T_BYTE;
569

570
  bool effective_indices_in_range = false;
571
  if (start_val->is_con() && step_val->is_con()) {
572
    int effective_min_index = start_val->get_con();
573
    int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
574
    effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
575
  }
576

577
  if (!do_wrap && !effective_indices_in_range) {
578
    // Disable instrinsification for unwrapped shuffle iota if start/step
579
    // values are non-constant OR if intermediate result overflows byte value range.
580
    return false;
581
  }
582

583
  if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)           ||
584
      !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)            ||
585
      !arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
586
      !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
587
    return false;
588
  }
589

590
  if (!do_wrap &&
591
      (!arch_supports_vector(Op_SubVB, num_elem, elem_bt, VecMaskNotUsed)       ||
592
      !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskNotUsed)  ||
593
      !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskNotUsed))) {
594
    return false;
595
  }
596

597
  bool step_multiply = !step_val->is_con() || !is_power_of_2(step_val->get_con());
598
  if ((step_multiply && !arch_supports_vector(Op_MulVB, num_elem, elem_bt, VecMaskNotUsed)) ||
599
      (!step_multiply && !arch_supports_vector(Op_LShiftVB, num_elem, elem_bt, VecMaskNotUsed))) {
600
    return false;
601
  }
602

603
  const Type * type_bt = Type::get_const_basic_type(elem_bt);
604
  const TypeVect * vt  = TypeVect::make(type_bt, num_elem);
605

606
  Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
607

608
  Node* start = argument(4);
609
  Node* step  = argument(5);
610

611
  if (step_multiply) {
612
    Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
613
    res = gvn().transform(VectorNode::make(Op_MulVB, res, bcast_step, vt));
614
  } else if (step_val->get_con() > 1) {
615
    Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
616
    Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
617
    res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
618
  }
619

620
  if (!start_val->is_con() || start_val->get_con() != 0) {
621
    Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
622
    res = gvn().transform(VectorNode::make(Op_AddVB, res, bcast_start, vt));
623
  }
624

625
  Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
626
  Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
627

628
  if (do_wrap)  {
629
    // Wrap the indices greater than lane count.
630
    res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
631
  } else {
632
    res = partially_wrap_indexes(res, num_elem, elem_bt);
633
  }
634

635
  ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
636
  const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
637

638
  // Wrap it up in VectorBox to keep object type information.
639
  res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
640
  set_result(res);
641
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
642
  return true;
643
}
644

645
// <E, M>
646
// long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
647
//                          int length, M m, VectorMaskOp<M> defaultImpl)
648
bool LibraryCallKit::inline_vector_mask_operation() {
649
  const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
650
  const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
651
  const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
652
  const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
653
  Node*              mask       = argument(4);
654

655
  if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
656
    return false; // dead code
657
  }
658

659
  if (!is_klass_initialized(mask_klass)) {
660
    log_if_needed("  ** klass argument not initialized");
661
    return false;
662
  }
663

664
  int num_elem = vlen->get_con();
665
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
666
  BasicType elem_bt = elem_type->basic_type();
667

668
  int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
669
  if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
670
    log_if_needed("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
671
                    mopc, num_elem, type2name(elem_bt));
672
    return false; // not supported
673
  }
674

675
  const Type* elem_ty = Type::get_const_basic_type(elem_bt);
676
  ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
677
  const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
678
  Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
679
  if (mask_vec == nullptr) {
680
    log_if_needed("  ** unbox failed mask=%s",
681
                      NodeClassNames[argument(4)->Opcode()]);
682
    return false;
683
  }
684

685
  if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
686
    mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
687
  }
688
  const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
689
  Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
690
  if (mopc != Op_VectorMaskToLong) {
691
    maskoper = ConvI2L(maskoper);
692
  }
693
  set_result(maskoper);
694

695
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
696
  return true;
697
}
698

699
// public static
700
// <V,
701
//  Sh extends VectorShuffle<E>,
702
//  E>
703
// V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
704
//                   Class<? extends Sh> shuffleClass, Sh s, int length,
705
//                   ShuffleToVectorOperation<V, Sh, E> defaultImpl)
706
bool LibraryCallKit::inline_vector_shuffle_to_vector() {
707
  const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
708
  const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
709
  const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
710
  Node*              shuffle       = argument(3);
711
  const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
712

713
  if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
714
    return false; // dead code
715
  }
716
  if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
717
    return false; // not enough info for intrinsification
718
  }
719
  if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
720
    log_if_needed("  ** klass argument not initialized");
721
    return false;
722
  }
723

724
  int num_elem = vlen->get_con();
725
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
726
  BasicType elem_bt = elem_type->basic_type();
727

728
  if (num_elem < 4) {
729
    return false;
730
  }
731

732
  int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
733
  // Make sure that cast is implemented to particular type/size combination.
734
  if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
735
    log_if_needed("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
736
        cast_vopc, num_elem, type2name(elem_bt));
737
    return false;
738
  }
739

740
  ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
741
  const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
742

743
  // Unbox shuffle with true flag to indicate its load shuffle to vector
744
  // shuffle is a byte array
745
  Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
746

747
  // cast byte to target element type
748
  shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
749

750
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
751
  const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
752

753
  // Box vector
754
  Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
755
  set_result(res);
756
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
757
  return true;
758
}
759

760
// public static
761
// <M,
762
//  S extends VectorSpecies<E>,
763
//  E>
764
// M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
765
//                    long bits, int mode, S s,
766
//                    BroadcastOperation<M, E, S> defaultImpl)
767
bool LibraryCallKit::inline_vector_frombits_coerced() {
768
  const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
769
  const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
770
  const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
771
  const TypeLong*    bits_type    = gvn().type(argument(3))->isa_long();
772
  // Mode argument determines the mode of operation it can take following values:-
773
  // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
774
  // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
775
  const TypeInt*     mode         = gvn().type(argument(5))->isa_int();
776

777
  if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
778
      bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
779
      !vlen->is_con() || !mode->is_con()) {
780
    log_if_needed("  ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
781
                    NodeClassNames[argument(0)->Opcode()],
782
                    NodeClassNames[argument(1)->Opcode()],
783
                    NodeClassNames[argument(2)->Opcode()],
784
                    NodeClassNames[argument(5)->Opcode()]);
785
    return false; // not enough info for intrinsification
786
  }
787

788
  if (!is_klass_initialized(vector_klass)) {
789
    log_if_needed("  ** klass argument not initialized");
790
    return false;
791
  }
792
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
793
  if (!elem_type->is_primitive_type()) {
794
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
795
    return false; // should be primitive type
796
  }
797
  BasicType elem_bt = elem_type->basic_type();
798
  int num_elem = vlen->get_con();
799
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
800
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
801

802
  bool is_mask = is_vector_mask(vbox_klass);
803
  int  bcast_mode = mode->get_con();
804
  VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
805
  int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
806

807
  if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
808
    log_if_needed("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
809
                    num_elem, type2name(elem_bt),
810
                    is_mask ? 1 : 0,
811
                    bcast_mode);
812
    return false; // not supported
813
  }
814

815
  Node* broadcast = nullptr;
816
  Node* bits = argument(3);
817
  Node* elem = bits;
818

819
  if (opc == Op_VectorLongToMask) {
820
    const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
821
    if (vt->isa_vectmask()) {
822
      broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
823
    } else {
824
      const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
825
      broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
826
      broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
827
    }
828
  } else {
829
    switch (elem_bt) {
830
      case T_BOOLEAN: // fall-through
831
      case T_BYTE:    // fall-through
832
      case T_SHORT:   // fall-through
833
      case T_CHAR:    // fall-through
834
      case T_INT: {
835
        elem = gvn().transform(new ConvL2INode(bits));
836
        break;
837
      }
838
      case T_DOUBLE: {
839
        elem = gvn().transform(new MoveL2DNode(bits));
840
        break;
841
      }
842
      case T_FLOAT: {
843
        bits = gvn().transform(new ConvL2INode(bits));
844
        elem = gvn().transform(new MoveI2FNode(bits));
845
        break;
846
      }
847
      case T_LONG: {
848
        // no conversion needed
849
        break;
850
      }
851
      default: fatal("%s", type2name(elem_bt));
852
    }
853
    broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
854
    broadcast = gvn().transform(broadcast);
855
  }
856

857
  Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
858
  set_result(box);
859
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
860
  return true;
861
}
862

863
static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
864
  assert(arr_type != nullptr, "unexpected");
865
  BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
866
  if (elem_bt == arr_elem_bt) {
867
    return true;
868
  } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
869
    // Load/store of short vector from/to char[] is supported
870
    return true;
871
  } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
872
    // Load/store of byte vector from/to boolean[] is supported
873
    return true;
874
  } else {
875
    return mismatched_ms;
876
  }
877
}
878

879
//  public static
880
//  <C,
881
//   VM extends VectorPayload,
882
//   E,
883
//   S extends VectorSpecies<E>>
884
//  VM load(Class<? extends VM> vmClass, Class<E> eClass,
885
//          int length,
886
//          Object base, long offset,            // Unsafe addressing
887
//          boolean fromSegment,
888
//          C container, long index, S s,        // Arguments for default implementation
889
//          LoadOperation<C, VM, S> defaultImpl) {
890
//  public static
891
//  <C,
892
//   V extends VectorPayload>
893
//  void store(Class<?> vClass, Class<?> eClass,
894
//             int length,
895
//             Object base, long offset,        // Unsafe addressing
896
//             boolean fromSegment,
897
//             V v, C container, long index,    // Arguments for default implementation
898
//             StoreVectorOperation<C, V> defaultImpl) {
899
bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
900
  const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
901
  const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
902
  const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
903
  const TypeInt*     from_ms      = gvn().type(argument(6))->isa_int();
904

905
  if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
906
      vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
907
    log_if_needed("  ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
908
                    NodeClassNames[argument(0)->Opcode()],
909
                    NodeClassNames[argument(1)->Opcode()],
910
                    NodeClassNames[argument(2)->Opcode()],
911
                    NodeClassNames[argument(6)->Opcode()]);
912
    return false; // not enough info for intrinsification
913
  }
914
  if (!is_klass_initialized(vector_klass)) {
915
    log_if_needed("  ** klass argument not initialized");
916
    return false;
917
  }
918

919
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
920
  if (!elem_type->is_primitive_type()) {
921
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
922
    return false; // should be primitive type
923
  }
924
  BasicType elem_bt = elem_type->basic_type();
925
  int num_elem = vlen->get_con();
926

927
  // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
928
  if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
929
    log_if_needed("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
930
                    is_store, is_store ? "store" : "load",
931
                    num_elem, type2name(elem_bt));
932
    return false; // not supported
933
  }
934

935
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
936
  bool is_mask = is_vector_mask(vbox_klass);
937

938
  Node* base = argument(3);
939
  Node* offset = ConvL2X(argument(4));
940

941
  // Save state and restore on bailout
942
  uint old_sp = sp();
943
  SafePointNode* old_map = clone_map();
944

945
  Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
946

947
  // The memory barrier checks are based on ones for unsafe access.
948
  // This is not 1-1 implementation.
949
  const Type *const base_type = gvn().type(base);
950

951
  const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
952
  const TypeAryPtr* arr_type = addr_type->isa_aryptr();
953

954
  const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
955
  const bool in_heap   = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
956

957
  const bool is_mixed_access = !in_heap && !in_native;
958

959
  const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
960

961
  const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
962

963
  // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
964
  // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
965
  // loading from or before storing to backing storage which is mandatory for semantic correctness of
966
  // big-endian memory layout.
967
  bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
968
      BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
969
                      arr_type->elem()->array_element_basic_type() != elem_bt);
970
  BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
971
  if (!is_java_primitive(mem_elem_bt)) {
972
    log_if_needed("  ** non-primitive array element type");
973
    return false;
974
  }
975
  int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
976
  if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
977
    log_if_needed("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
978
                    is_store, is_store ? "store" : "load",
979
                    num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
980
    set_map(old_map);
981
    set_sp(old_sp);
982
    return false;
983
  }
984

985
  // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
986
  if (mismatched_ms) {
987
    if (is_store) {
988
      if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
989
          || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
990
        log_if_needed("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
991
                        is_store, "store",
992
                        num_elem, type2name(elem_bt));
993
        set_map(old_map);
994
        set_sp(old_sp);
995
        return false; // not supported
996
      }
997
    } else {
998
      if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
999
          || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
1000
        log_if_needed("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1001
                        is_store, "load",
1002
                        mem_num_elem, type2name(mem_elem_bt));
1003
        set_map(old_map);
1004
        set_sp(old_sp);
1005
        return false; // not supported
1006
      }
1007
    }
1008
  }
1009
  if (is_mask) {
1010
    if (!is_store) {
1011
      if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1012
        set_map(old_map);
1013
        set_sp(old_sp);
1014
        return false; // not supported
1015
      }
1016
    } else {
1017
      if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1018
        set_map(old_map);
1019
        set_sp(old_sp);
1020
        return false; // not supported
1021
      }
1022
    }
1023
  }
1024

1025
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1026

1027
  if (needs_cpu_membar) {
1028
    insert_mem_bar(Op_MemBarCPUOrder);
1029
  }
1030

1031
  if (is_store) {
1032
    Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1033
    if (val == nullptr) {
1034
      set_map(old_map);
1035
      set_sp(old_sp);
1036
      return false; // operand unboxing failed
1037
    }
1038
    set_all_memory(reset_memory());
1039

1040
    // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1041
    int store_num_elem = num_elem;
1042
    if (mismatched_ms) {
1043
      store_num_elem = mem_num_elem;
1044
      const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
1045
      val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1046
    }
1047
    if (is_mask) {
1048
      val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1049
    }
1050
    Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1051
    set_memory(vstore, addr_type);
1052
  } else {
1053
    // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1054
    Node* vload = nullptr;
1055
    if (mismatched_ms) {
1056
      vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1057
      const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1058
      vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1059
    } else {
1060
      // Special handle for masks
1061
      if (is_mask) {
1062
        vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1063
        vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1064
      } else {
1065
        vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1066
      }
1067
    }
1068
    Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1069
    set_result(box);
1070
  }
1071

1072
  destruct_map_clone(old_map);
1073

1074
  if (needs_cpu_membar) {
1075
    insert_mem_bar(Op_MemBarCPUOrder);
1076
  }
1077

1078
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1079
  return true;
1080
}
1081

1082
//  public static
1083
//  <C,
1084
//   V extends Vector<?>,
1085
//   E,
1086
//   S extends VectorSpecies<E>,
1087
//   M extends VectorMask<E>>
1088
//  V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1089
//               int length, Object base, long offset,          // Unsafe addressing
1090
//               boolean fromSegment,
1091
//               M m, int offsetInRange,
1092
//               C container, long index, S s,                  // Arguments for default implementation
1093
//               LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1094
//  public static
1095
//  <C,
1096
//   V extends Vector<E>,
1097
//   M extends VectorMask<E>,
1098
//   E>
1099
//  void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1100
//                   int length,
1101
//                   Object base, long offset,                  // Unsafe addressing
1102
//                   boolean fromSegment,
1103
//                   V v, M m, C container, long index,         // Arguments for default implementation
1104
//                   StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1105

1106
bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1107
  const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1108
  const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1109
  const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1110
  const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1111
  const TypeInt*     from_ms      = gvn().type(argument(7))->isa_int();
1112

1113
  if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1114
      vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1115
      elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
1116
    log_if_needed("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1117
                    NodeClassNames[argument(0)->Opcode()],
1118
                    NodeClassNames[argument(1)->Opcode()],
1119
                    NodeClassNames[argument(2)->Opcode()],
1120
                    NodeClassNames[argument(3)->Opcode()],
1121
                    NodeClassNames[argument(7)->Opcode()]);
1122
    return false; // not enough info for intrinsification
1123
  }
1124
  if (!is_klass_initialized(vector_klass)) {
1125
    log_if_needed("  ** klass argument not initialized");
1126
    return false;
1127
  }
1128

1129
  if (!is_klass_initialized(mask_klass)) {
1130
    log_if_needed("  ** mask klass argument not initialized");
1131
    return false;
1132
  }
1133

1134
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1135
  if (!elem_type->is_primitive_type()) {
1136
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
1137
    return false; // should be primitive type
1138
  }
1139

1140
  BasicType elem_bt = elem_type->basic_type();
1141
  int num_elem = vlen->get_con();
1142

1143
  Node* base = argument(4);
1144
  Node* offset = ConvL2X(argument(5));
1145

1146
  // Save state and restore on bailout
1147
  uint old_sp = sp();
1148
  SafePointNode* old_map = clone_map();
1149

1150
  Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1151
  const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1152
  const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1153

1154
  bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1155
  BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1156
  // If there is no consistency between array and vector element types, it must be special byte array case
1157
  if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1158
    log_if_needed("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1159
                    is_store, is_store ? "storeMasked" : "loadMasked",
1160
                    num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1161
    set_map(old_map);
1162
    set_sp(old_sp);
1163
    return false;
1164
  }
1165

1166
  int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1167
  BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1168
  bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1169
                                                mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1170

1171
  // If current arch does not support the predicated operations, we have to bail
1172
  // out when current case uses the predicate feature.
1173
  if (!supports_predicate) {
1174
    bool needs_predicate = false;
1175
    if (is_store) {
1176
      // Masked vector store always uses the predicated store.
1177
      needs_predicate = true;
1178
    } else {
1179
      // Masked vector load with IOOBE always uses the predicated load.
1180
      const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1181
      if (!offset_in_range->is_con()) {
1182
        log_if_needed("  ** missing constant: offsetInRange=%s",
1183
                        NodeClassNames[argument(8)->Opcode()]);
1184
        set_map(old_map);
1185
        set_sp(old_sp);
1186
        return false;
1187
      }
1188
      needs_predicate = (offset_in_range->get_con() == 0);
1189
    }
1190

1191
    if (needs_predicate) {
1192
      log_if_needed("  ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1193
                      is_store ? "storeMasked" : "loadMasked",
1194
                      num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1195
      set_map(old_map);
1196
      set_sp(old_sp);
1197
      return false;
1198
    }
1199
  }
1200

1201
  // This only happens for masked vector load. If predicate is not supported, then check whether
1202
  // the normal vector load and blend operations are supported by backend.
1203
  if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1204
      !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1205
    log_if_needed("  ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1206
                    num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1207
    set_map(old_map);
1208
    set_sp(old_sp);
1209
    return false;
1210
  }
1211

1212
  // Since we are using byte array, we need to double check that the vector reinterpret operation
1213
  // with byte type is supported by backend.
1214
  if (mismatched_ms) {
1215
    if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1216
      log_if_needed("  ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1217
                      is_store, is_store ? "storeMasked" : "loadMasked",
1218
                      num_elem, type2name(elem_bt));
1219
      set_map(old_map);
1220
      set_sp(old_sp);
1221
      return false;
1222
    }
1223
  }
1224

1225
  // Since it needs to unbox the mask, we need to double check that the related load operations
1226
  // for mask are supported by backend.
1227
  if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1228
    log_if_needed("  ** not supported: arity=%d op=%s vlen=%d etype=%s",
1229
                      is_store, is_store ? "storeMasked" : "loadMasked",
1230
                      num_elem, type2name(elem_bt));
1231
    set_map(old_map);
1232
    set_sp(old_sp);
1233
    return false;
1234
  }
1235

1236
  // Can base be null? Otherwise, always on-heap access.
1237
  bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1238
  if (can_access_non_heap) {
1239
    insert_mem_bar(Op_MemBarCPUOrder);
1240
  }
1241

1242
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1243
  ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1244
  assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1245
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1246
  const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1247

1248
  Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1249
  if (mask == nullptr) {
1250
    log_if_needed("  ** unbox failed mask=%s",
1251
                    is_store ? NodeClassNames[argument(9)->Opcode()]
1252
                             : NodeClassNames[argument(8)->Opcode()]);
1253
    set_map(old_map);
1254
    set_sp(old_sp);
1255
    return false;
1256
  }
1257

1258
  if (is_store) {
1259
    Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1260
    if (val == nullptr) {
1261
      log_if_needed("  ** unbox failed vector=%s",
1262
                      NodeClassNames[argument(8)->Opcode()]);
1263
      set_map(old_map);
1264
      set_sp(old_sp);
1265
      return false; // operand unboxing failed
1266
    }
1267
    set_all_memory(reset_memory());
1268

1269
    if (mismatched_ms) {
1270
      // Reinterpret the incoming vector to byte vector.
1271
      const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1272
      val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1273
      // Reinterpret the vector mask to byte type.
1274
      const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1275
      const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1276
      mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1277
    }
1278
    Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1279
    set_memory(vstore, addr_type);
1280
  } else {
1281
    Node* vload = nullptr;
1282

1283
    if (mismatched_ms) {
1284
      // Reinterpret the vector mask to byte type.
1285
      const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1286
      const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1287
      mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1288
    }
1289

1290
    if (supports_predicate) {
1291
      // Generate masked load vector node if predicate feature is supported.
1292
      const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1293
      vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1294
    } else {
1295
      // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1296
      Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1297
      zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1298
      vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1299
      vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1300
    }
1301

1302
    if (mismatched_ms) {
1303
      const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1304
      vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1305
    }
1306

1307
    Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1308
    set_result(box);
1309
  }
1310

1311
  destruct_map_clone(old_map);
1312

1313
  if (can_access_non_heap) {
1314
    insert_mem_bar(Op_MemBarCPUOrder);
1315
  }
1316

1317
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1318
  return true;
1319
}
1320

1321
// <C,
1322
//  V extends Vector<?>,
1323
//  W extends Vector<Integer>,
1324
//  S extends VectorSpecies<E>,
1325
//  M extends VectorMask<E>,
1326
//  E>
1327
// V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1328
//               Class<? extends Vector<Integer>> vectorIndexClass,
1329
//               Object base, long offset, // Unsafe addressing
1330
//               W index_vector, M m,
1331
//               C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1332
//               LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1333
//
1334
//  <C,
1335
//   V extends Vector<E>,
1336
//   W extends Vector<Integer>,
1337
//   M extends VectorMask<E>,
1338
//   E>
1339
//  void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1340
//                    int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset,    // Unsafe addressing
1341
//                    W index_vector, V v, M m,
1342
//                    C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1343
//                    StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1344
//
1345
bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1346
  const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
1347
  const TypeInstPtr* mask_klass       = gvn().type(argument(1))->isa_instptr();
1348
  const TypeInstPtr* elem_klass       = gvn().type(argument(2))->isa_instptr();
1349
  const TypeInt*     vlen             = gvn().type(argument(3))->isa_int();
1350
  const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1351

1352
  if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1353
      vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1354
    log_if_needed("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1355
                    NodeClassNames[argument(0)->Opcode()],
1356
                    NodeClassNames[argument(2)->Opcode()],
1357
                    NodeClassNames[argument(3)->Opcode()],
1358
                    NodeClassNames[argument(4)->Opcode()]);
1359
    return false; // not enough info for intrinsification
1360
  }
1361

1362
  if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1363
    log_if_needed("  ** klass argument not initialized");
1364
    return false;
1365
  }
1366

1367
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1368
  if (!elem_type->is_primitive_type()) {
1369
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
1370
    return false; // should be primitive type
1371
  }
1372

1373
  BasicType elem_bt = elem_type->basic_type();
1374
  int num_elem = vlen->get_con();
1375

1376
  const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1377
  bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1378
  if (is_masked_op) {
1379
    if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1380
      log_if_needed("  ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1381
      return false; // not enough info for intrinsification
1382
    }
1383

1384
    if (!is_klass_initialized(mask_klass)) {
1385
      log_if_needed("  ** mask klass argument not initialized");
1386
      return false;
1387
    }
1388

1389
    if (vmask_type->maybe_null()) {
1390
      log_if_needed("  ** null mask values are not allowed for masked op");
1391
      return false;
1392
    }
1393

1394
    // Check whether the predicated gather/scatter node is supported by architecture.
1395
    VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1396
    if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1397
      log_if_needed("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1398
                      is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1399
                      num_elem, type2name(elem_bt));
1400
      return false; // not supported
1401
    }
1402
  } else {
1403
    // Check whether the normal gather/scatter node is supported for non-masked operation.
1404
    if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1405
      log_if_needed("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1406
                      is_scatter, is_scatter ? "scatter" : "gather",
1407
                      num_elem, type2name(elem_bt));
1408
      return false; // not supported
1409
    }
1410
  }
1411

1412
  // Check that the vector holding indices is supported by architecture
1413
  // For sub-word gathers expander receive index array.
1414
  if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1415
      log_if_needed("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1416
                      is_scatter, is_scatter ? "scatter" : "gather",
1417
                      num_elem, is_masked_op ? 1 : 0);
1418
      return false; // not supported
1419
  }
1420

1421
  Node* base = argument(5);
1422
  Node* offset = ConvL2X(argument(6));
1423

1424
  // Save state and restore on bailout
1425
  uint old_sp = sp();
1426
  SafePointNode* old_map = clone_map();
1427

1428
  Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1429

1430
  const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1431
  const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1432

1433
  // The array must be consistent with vector type
1434
  if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1435
    log_if_needed("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1436
                    is_scatter, is_scatter ? "scatter" : "gather",
1437
                    num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1438
    set_map(old_map);
1439
    set_sp(old_sp);
1440
    return false;
1441
  }
1442

1443
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1444
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1445
  ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1446
  if (vbox_idx_klass == nullptr) {
1447
    set_map(old_map);
1448
    set_sp(old_sp);
1449
    return false;
1450
  }
1451

1452
  Node* index_vect = nullptr;
1453
  const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1454
  if (!is_subword_type(elem_bt)) {
1455
    index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1456
    if (index_vect == nullptr) {
1457
      set_map(old_map);
1458
      set_sp(old_sp);
1459
      return false;
1460
    }
1461
  }
1462

1463
  Node* mask = nullptr;
1464
  if (is_masked_op) {
1465
    ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1466
    const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1467
    mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1468
    if (mask == nullptr) {
1469
      log_if_needed("  ** unbox failed mask=%s",
1470
                    is_scatter ? NodeClassNames[argument(10)->Opcode()]
1471
                               : NodeClassNames[argument(9)->Opcode()]);
1472
      set_map(old_map);
1473
      set_sp(old_sp);
1474
      return false;
1475
    }
1476
  }
1477

1478
  const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1479
  if (is_scatter) {
1480
    Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1481
    if (val == nullptr) {
1482
      set_map(old_map);
1483
      set_sp(old_sp);
1484
      return false; // operand unboxing failed
1485
    }
1486
    set_all_memory(reset_memory());
1487

1488
    Node* vstore = nullptr;
1489
    if (mask != nullptr) {
1490
      vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1491
    } else {
1492
      vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1493
    }
1494
    set_memory(vstore, addr_type);
1495
  } else {
1496
    Node* vload = nullptr;
1497
    Node* index    = argument(11);
1498
    Node* indexMap = argument(12);
1499
    Node* indexM   = argument(13);
1500
    if (mask != nullptr) {
1501
      if (is_subword_type(elem_bt)) {
1502
        Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1503
        vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1504
      } else {
1505
        vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1506
      }
1507
    } else {
1508
      if (is_subword_type(elem_bt)) {
1509
        Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1510
        vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1511
      } else {
1512
        vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1513
      }
1514
    }
1515
    Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1516
    set_result(box);
1517
  }
1518

1519
  destruct_map_clone(old_map);
1520

1521
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1522
  return true;
1523
}
1524

1525
// public static
1526
// <V extends Vector<E>,
1527
//  M extends VectorMask<E>,
1528
//  E>
1529
// long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1530
//                       Class<E> elementType, int length, V v, M m,
1531
//                       ReductionOperation<V, M> defaultImpl)
1532
bool LibraryCallKit::inline_vector_reduction() {
1533
  const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1534
  const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1535
  const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1536
  const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1537
  const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1538

1539
  if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1540
      !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1541
    log_if_needed("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1542
                    NodeClassNames[argument(0)->Opcode()],
1543
                    NodeClassNames[argument(1)->Opcode()],
1544
                    NodeClassNames[argument(3)->Opcode()],
1545
                    NodeClassNames[argument(4)->Opcode()]);
1546
    return false; // not enough info for intrinsification
1547
  }
1548
  if (!is_klass_initialized(vector_klass)) {
1549
    log_if_needed("  ** klass argument not initialized");
1550
    return false;
1551
  }
1552
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1553
  if (!elem_type->is_primitive_type()) {
1554
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
1555
    return false; // should be primitive type
1556
  }
1557

1558
  const Type* vmask_type = gvn().type(argument(6));
1559
  bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1560
  if (is_masked_op) {
1561
    if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1562
      log_if_needed("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1563
      return false; // not enough info for intrinsification
1564
    }
1565

1566
    if (!is_klass_initialized(mask_klass)) {
1567
      log_if_needed("  ** mask klass argument not initialized");
1568
      return false;
1569
    }
1570

1571
    if (vmask_type->maybe_null()) {
1572
      log_if_needed("  ** null mask values are not allowed for masked op");
1573
      return false;
1574
    }
1575
  }
1576

1577
  BasicType elem_bt = elem_type->basic_type();
1578
  int num_elem = vlen->get_con();
1579
  int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1580
  int sopc = ReductionNode::opcode(opc, elem_bt);
1581

1582
  // When using mask, mask use type needs to be VecMaskUseLoad.
1583
  if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1584
    log_if_needed("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1585
                    sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1586
    return false;
1587
  }
1588

1589
  // Return true if current platform has implemented the masked operation with predicate feature.
1590
  bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1591
  if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1592
    log_if_needed("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1593
                    sopc, num_elem, type2name(elem_bt));
1594
    return false;
1595
  }
1596

1597
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1598
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1599

1600
  Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1601
  if (opd == nullptr) {
1602
    return false; // operand unboxing failed
1603
  }
1604

1605
  Node* mask = nullptr;
1606
  if (is_masked_op) {
1607
    ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1608
    assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1609
    const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1610
    mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1611
    if (mask == nullptr) {
1612
      log_if_needed("  ** unbox failed mask=%s",
1613
                      NodeClassNames[argument(6)->Opcode()]);
1614
      return false;
1615
    }
1616
  }
1617

1618
  Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1619
  Node* value = opd;
1620

1621
  assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1622
  if (mask != nullptr && !use_predicate) {
1623
    Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1624
    value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1625
  }
1626

1627
  // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1628
  // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1629
  value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1630

1631
  if (mask != nullptr && use_predicate) {
1632
    value->add_req(mask);
1633
    value->add_flag(Node::Flag_is_predicated_vector);
1634
  }
1635

1636
  value = gvn().transform(value);
1637

1638
  Node* bits = nullptr;
1639
  switch (elem_bt) {
1640
    case T_BYTE:
1641
    case T_SHORT:
1642
    case T_INT: {
1643
      bits = gvn().transform(new ConvI2LNode(value));
1644
      break;
1645
    }
1646
    case T_FLOAT: {
1647
      value = gvn().transform(new MoveF2INode(value));
1648
      bits  = gvn().transform(new ConvI2LNode(value));
1649
      break;
1650
    }
1651
    case T_DOUBLE: {
1652
      bits = gvn().transform(new MoveD2LNode(value));
1653
      break;
1654
    }
1655
    case T_LONG: {
1656
      bits = value; // no conversion needed
1657
      break;
1658
    }
1659
    default: fatal("%s", type2name(elem_bt));
1660
  }
1661
  set_result(bits);
1662
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1663
  return true;
1664
}
1665

1666
// public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1667
//                                V v1, V v2,
1668
//                                BiFunction<V, V, Boolean> defaultImpl)
1669
//
1670
bool LibraryCallKit::inline_vector_test() {
1671
  const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1672
  const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1673
  const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1674
  const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1675

1676
  if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1677
      !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1678
    log_if_needed("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1679
                    NodeClassNames[argument(0)->Opcode()],
1680
                    NodeClassNames[argument(1)->Opcode()],
1681
                    NodeClassNames[argument(2)->Opcode()],
1682
                    NodeClassNames[argument(3)->Opcode()]);
1683
    return false; // not enough info for intrinsification
1684
  }
1685
  if (!is_klass_initialized(vector_klass)) {
1686
    log_if_needed("  ** klass argument not initialized");
1687
    return false;
1688
  }
1689
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1690
  if (!elem_type->is_primitive_type()) {
1691
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
1692
    return false; // should be primitive type
1693
  }
1694
  BasicType elem_bt = elem_type->basic_type();
1695
  int num_elem = vlen->get_con();
1696
  BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1697
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1698
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1699

1700
  if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1701
    log_if_needed("  ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1702
                    cond->get_con(), num_elem, type2name(elem_bt),
1703
                    is_vector_mask(vbox_klass));
1704
    return false;
1705
  }
1706

1707
  Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1708
  Node* opd2;
1709
  if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1710
                                                opd1->bottom_type()->isa_vectmask())) {
1711
    opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1712
  } else {
1713
    opd2 = opd1;
1714
  }
1715
  if (opd1 == nullptr || opd2 == nullptr) {
1716
    return false; // operand unboxing failed
1717
  }
1718

1719
  Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1720
  BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1721
                                                 opd1->bottom_type()->isa_vectmask(), num_elem);
1722
  Node* bol = gvn().transform(new BoolNode(cmp, test));
1723
  Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1724

1725
  set_result(res);
1726
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1727
  return true;
1728
}
1729

1730
// public static
1731
// <V extends Vector<E>,
1732
//  M extends VectorMask<E>,
1733
//  E>
1734
// V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1735
//         V v1, V v2, M m,
1736
//         VectorBlendOp<V, M, E> defaultImpl)
1737
bool LibraryCallKit::inline_vector_blend() {
1738
  const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1739
  const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1740
  const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1741
  const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1742

1743
  if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1744
    return false; // dead code
1745
  }
1746
  if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1747
      elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1748
    log_if_needed("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1749
                    NodeClassNames[argument(0)->Opcode()],
1750
                    NodeClassNames[argument(1)->Opcode()],
1751
                    NodeClassNames[argument(2)->Opcode()],
1752
                    NodeClassNames[argument(3)->Opcode()]);
1753
    return false; // not enough info for intrinsification
1754
  }
1755
  if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1756
    log_if_needed("  ** klass argument not initialized");
1757
    return false;
1758
  }
1759
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1760
  if (!elem_type->is_primitive_type()) {
1761
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
1762
    return false; // should be primitive type
1763
  }
1764
  BasicType elem_bt = elem_type->basic_type();
1765
  BasicType mask_bt = elem_bt;
1766
  int num_elem = vlen->get_con();
1767

1768
  if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1769
    log_if_needed("  ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1770
                    num_elem, type2name(elem_bt));
1771
    return false; // not supported
1772
  }
1773
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1774
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1775

1776
  ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1777
  const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1778

1779
  Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1780
  Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1781
  Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1782

1783
  if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1784
    return false; // operand unboxing failed
1785
  }
1786

1787
  Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1788

1789
  Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1790
  set_result(box);
1791
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1792
  return true;
1793
}
1794

1795
//  public static
1796
//  <V extends Vector<E>,
1797
//   M extends VectorMask<E>,
1798
//   E>
1799
//  M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1800
//            V v1, V v2, M m,
1801
//            VectorCompareOp<V,M> defaultImpl)
1802
bool LibraryCallKit::inline_vector_compare() {
1803
  const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1804
  const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1805
  const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1806
  const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1807
  const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1808

1809
  if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1810
    return false; // dead code
1811
  }
1812
  if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1813
      elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1814
    log_if_needed("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1815
                    NodeClassNames[argument(0)->Opcode()],
1816
                    NodeClassNames[argument(1)->Opcode()],
1817
                    NodeClassNames[argument(2)->Opcode()],
1818
                    NodeClassNames[argument(3)->Opcode()],
1819
                    NodeClassNames[argument(4)->Opcode()]);
1820
    return false; // not enough info for intrinsification
1821
  }
1822
  if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1823
    log_if_needed("  ** klass argument not initialized");
1824
    return false;
1825
  }
1826
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1827
  if (!elem_type->is_primitive_type()) {
1828
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
1829
    return false; // should be primitive type
1830
  }
1831

1832
  int num_elem = vlen->get_con();
1833
  BasicType elem_bt = elem_type->basic_type();
1834
  BasicType mask_bt = elem_bt;
1835

1836
  if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1837
    if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1838
      log_if_needed("  ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1839
                      cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1840
      return false;
1841
    }
1842
  }
1843

1844
  if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1845
    log_if_needed("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1846
                    cond->get_con(), num_elem, type2name(elem_bt));
1847
    return false;
1848
  }
1849

1850
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1851
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1852

1853
  ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1854
  const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1855

1856
  Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1857
  Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1858

1859
  bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1860
  Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1861
  if (is_masked_op && mask == nullptr) {
1862
    log_if_needed("  ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1863
                    cond->get_con(), num_elem, type2name(elem_bt));
1864
    return false;
1865
  }
1866

1867
  bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1868
  if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1869
    log_if_needed("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1870
                    cond->get_con(), num_elem, type2name(elem_bt));
1871
    return false;
1872
  }
1873

1874
  if (v1 == nullptr || v2 == nullptr) {
1875
    return false; // operand unboxing failed
1876
  }
1877
  BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1878
  ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1879

1880
  const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1881
  Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1882

1883
  if (is_masked_op) {
1884
    if (use_predicate) {
1885
      operation->add_req(mask);
1886
      operation->add_flag(Node::Flag_is_predicated_vector);
1887
    } else {
1888
      operation = gvn().transform(operation);
1889
      operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1890
    }
1891
  }
1892

1893
  operation = gvn().transform(operation);
1894

1895
  Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1896
  set_result(box);
1897
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1898
  return true;
1899
}
1900

1901
// public static
1902
// <V extends Vector<E>,
1903
//  Sh extends VectorShuffle<E>,
1904
//  M extends VectorMask<E>,
1905
//  E>
1906
// V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1907
//               V v1, Sh sh, M m,
1908
//               VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1909
bool LibraryCallKit::inline_vector_rearrange() {
1910
  const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
1911
  const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1912
  const TypeInstPtr* mask_klass    = gvn().type(argument(2))->isa_instptr();
1913
  const TypeInstPtr* elem_klass    = gvn().type(argument(3))->isa_instptr();
1914
  const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
1915

1916
  if (vector_klass == nullptr  || shuffle_klass == nullptr ||  elem_klass == nullptr || vlen == nullptr) {
1917
    return false; // dead code
1918
  }
1919
  if (shuffle_klass->const_oop() == nullptr ||
1920
      vector_klass->const_oop()  == nullptr ||
1921
      elem_klass->const_oop()    == nullptr ||
1922
      !vlen->is_con()) {
1923
    log_if_needed("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1924
                    NodeClassNames[argument(0)->Opcode()],
1925
                    NodeClassNames[argument(1)->Opcode()],
1926
                    NodeClassNames[argument(3)->Opcode()],
1927
                    NodeClassNames[argument(4)->Opcode()]);
1928
    return false; // not enough info for intrinsification
1929
  }
1930
  if (!is_klass_initialized(vector_klass)  ||
1931
      !is_klass_initialized(shuffle_klass)) {
1932
    log_if_needed("  ** klass argument not initialized");
1933
    return false;
1934
  }
1935
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1936
  if (!elem_type->is_primitive_type()) {
1937
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
1938
    return false; // should be primitive type
1939
  }
1940
  BasicType elem_bt = elem_type->basic_type();
1941
  BasicType shuffle_bt = elem_bt;
1942
  int num_elem = vlen->get_con();
1943

1944
  if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1945
    log_if_needed("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1946
                    num_elem, type2name(elem_bt));
1947
    return false; // not supported
1948
  }
1949

1950
  bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1951
  bool use_predicate = is_masked_op;
1952
  if (is_masked_op &&
1953
      (mask_klass == nullptr ||
1954
       mask_klass->const_oop() == nullptr ||
1955
       !is_klass_initialized(mask_klass))) {
1956
    log_if_needed("  ** mask_klass argument not initialized");
1957
  }
1958
  VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1959
  if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1960
    use_predicate = false;
1961
    if(!is_masked_op ||
1962
       (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1963
        !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)     ||
1964
        !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1965
      log_if_needed("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1966
                      num_elem, type2name(elem_bt));
1967
      return false; // not supported
1968
    }
1969
  }
1970
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1971
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1972

1973
  ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1974
  const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1975

1976
  Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1977
  Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1978

1979
  if (v1 == nullptr || shuffle == nullptr) {
1980
    return false; // operand unboxing failed
1981
  }
1982

1983
  Node* mask = nullptr;
1984
  if (is_masked_op) {
1985
    ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1986
    const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1987
    mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1988
    if (mask == nullptr) {
1989
      log_if_needed("  ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1990
                      num_elem, type2name(elem_bt));
1991
      return false;
1992
    }
1993
  }
1994

1995
  Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1996
  if (is_masked_op) {
1997
    if (use_predicate) {
1998
      rearrange->add_req(mask);
1999
      rearrange->add_flag(Node::Flag_is_predicated_vector);
2000
    } else {
2001
      const TypeVect* vt = v1->bottom_type()->is_vect();
2002
      rearrange = gvn().transform(rearrange);
2003
      Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2004
      Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2005
      rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2006
    }
2007
  }
2008
  rearrange = gvn().transform(rearrange);
2009

2010
  Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2011
  set_result(box);
2012
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2013
  return true;
2014
}
2015

2016
static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2017
  address addr = nullptr;
2018
  assert(UseVectorStubs, "sanity");
2019
  assert(name_ptr != nullptr, "unexpected");
2020
  assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2021
  int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2022

2023
  switch(bits) {
2024
    case 64:  //fallthough
2025
    case 128: //fallthough
2026
    case 256: //fallthough
2027
    case 512:
2028
      if (bt == T_FLOAT) {
2029
        snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2030
        addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2031
      } else {
2032
        assert(bt == T_DOUBLE, "must be FP type only");
2033
        snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2034
        addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2035
      }
2036
      break;
2037
    default:
2038
      snprintf(name_ptr, name_len, "invalid");
2039
      addr = nullptr;
2040
      Unimplemented();
2041
      break;
2042
  }
2043

2044
  return addr;
2045
}
2046

2047
Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2048
  assert(UseVectorStubs, "sanity");
2049
  assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2050
  assert(opd1 != nullptr, "must not be null");
2051
  const TypeVect* vt = TypeVect::make(bt, num_elem);
2052
  const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2053
  char name[100] = "";
2054

2055
  // Get address for svml method.
2056
  address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2057

2058
  if (addr == nullptr) {
2059
    return nullptr;
2060
  }
2061

2062
  assert(name[0] != '\0', "name must not be null");
2063
  Node* operation = make_runtime_call(RC_VECTOR,
2064
                                      call_type,
2065
                                      addr,
2066
                                      name,
2067
                                      TypePtr::BOTTOM,
2068
                                      opd1,
2069
                                      opd2);
2070
  return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2071
}
2072

2073
//  public static
2074
//  <V extends Vector<E>,
2075
//   M extends VectorMask<E>,
2076
//   E>
2077
//  V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2078
//                 Class<E> elementType, int length,
2079
//                 V v, int n, M m,
2080
//                 VectorBroadcastIntOp<V, M> defaultImpl)
2081
bool LibraryCallKit::inline_vector_broadcast_int() {
2082
  const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2083
  const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2084
  const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2085
  const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2086
  const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2087

2088
  if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2089
    return false; // dead code
2090
  }
2091
  if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2092
    log_if_needed("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2093
                    NodeClassNames[argument(0)->Opcode()],
2094
                    NodeClassNames[argument(1)->Opcode()],
2095
                    NodeClassNames[argument(3)->Opcode()],
2096
                    NodeClassNames[argument(4)->Opcode()]);
2097
    return false; // not enough info for intrinsification
2098
  }
2099
  if (!is_klass_initialized(vector_klass)) {
2100
    log_if_needed("  ** klass argument not initialized");
2101
    return false;
2102
  }
2103

2104
  const Type* vmask_type = gvn().type(argument(7));
2105
  bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2106
  if (is_masked_op) {
2107
    if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2108
      log_if_needed("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2109
      return false; // not enough info for intrinsification
2110
    }
2111

2112
    if (!is_klass_initialized(mask_klass)) {
2113
      log_if_needed("  ** mask klass argument not initialized");
2114
      return false;
2115
    }
2116

2117
    if (vmask_type->maybe_null()) {
2118
      log_if_needed("  ** null mask values are not allowed for masked op");
2119
      return false;
2120
    }
2121
  }
2122

2123
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2124
  if (!elem_type->is_primitive_type()) {
2125
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
2126
    return false; // should be primitive type
2127
  }
2128

2129
  int num_elem = vlen->get_con();
2130
  BasicType elem_bt = elem_type->basic_type();
2131
  int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2132

2133
  bool is_shift  = VectorNode::is_shift_opcode(opc);
2134
  bool is_rotate = VectorNode::is_rotate_opcode(opc);
2135

2136
  if (opc == 0 || (!is_shift && !is_rotate)) {
2137
    log_if_needed("  ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2138
    return false; // operation not supported
2139
  }
2140

2141
  int sopc = VectorNode::opcode(opc, elem_bt);
2142
  if (sopc == 0) {
2143
    log_if_needed("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2144
    return false; // operation not supported
2145
  }
2146

2147
  Node* cnt  = argument(6);
2148
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2149
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2150
  const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2151

2152
  // If CPU supports vector constant rotate instructions pass it directly
2153
  bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2154
                         Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2155
  bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2156

2157
  VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2158
  bool use_predicate = is_masked_op;
2159

2160
  if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2161
    use_predicate = false;
2162
    if (!is_masked_op ||
2163
        (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2164
         !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2165

2166
      log_if_needed("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2167
                      sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2168
      return false; // not supported
2169
    }
2170
  }
2171

2172
  Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2173
  Node* opd2 = nullptr;
2174
  if (is_shift) {
2175
    opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2176
  } else {
2177
    assert(is_rotate, "unexpected operation");
2178
    if (!is_const_rotate) {
2179
      const Type * type_bt = Type::get_const_basic_type(elem_bt);
2180
      cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2181
      opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2182
    } else {
2183
      // Constant shift value.
2184
      opd2 = cnt;
2185
    }
2186
  }
2187

2188
  if (opd1 == nullptr || opd2 == nullptr) {
2189
    return false;
2190
  }
2191

2192
  Node* mask = nullptr;
2193
  if (is_masked_op) {
2194
    ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2195
    const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2196
    mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2197
    if (mask == nullptr) {
2198
      log_if_needed("  ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2199
      return false;
2200
    }
2201
  }
2202

2203
  Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2204
  if (is_masked_op && mask != nullptr) {
2205
    if (use_predicate) {
2206
      operation->add_req(mask);
2207
      operation->add_flag(Node::Flag_is_predicated_vector);
2208
    } else {
2209
      operation = gvn().transform(operation);
2210
      operation = new VectorBlendNode(opd1, operation, mask);
2211
    }
2212
  }
2213
  operation = gvn().transform(operation);
2214
  Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2215
  set_result(vbox);
2216
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2217
  return true;
2218
}
2219

2220
// public static <VOUT extends VectorPayload,
2221
//                 VIN extends VectorPayload,
2222
//                   S extends VectorSpecies>
2223
// VOUT convert(int oprId,
2224
//           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2225
//           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
2226
//           VIN v, S s,
2227
//           VectorConvertOp<VOUT, VIN, S> defaultImpl)
2228
//
2229
bool LibraryCallKit::inline_vector_convert() {
2230
  const TypeInt*     opr               = gvn().type(argument(0))->isa_int();
2231

2232
  const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2233
  const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->isa_instptr();
2234
  const TypeInt*     vlen_from         = gvn().type(argument(3))->isa_int();
2235

2236
  const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->isa_instptr();
2237
  const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->isa_instptr();
2238
  const TypeInt*     vlen_to           = gvn().type(argument(6))->isa_int();
2239

2240
  if (opr == nullptr ||
2241
      vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2242
      vector_klass_to   == nullptr || elem_klass_to   == nullptr || vlen_to   == nullptr) {
2243
    return false; // dead code
2244
  }
2245
  if (!opr->is_con() ||
2246
      vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2247
      vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2248
    log_if_needed("  ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2249
                    NodeClassNames[argument(0)->Opcode()],
2250
                    NodeClassNames[argument(1)->Opcode()],
2251
                    NodeClassNames[argument(2)->Opcode()],
2252
                    NodeClassNames[argument(3)->Opcode()],
2253
                    NodeClassNames[argument(4)->Opcode()],
2254
                    NodeClassNames[argument(5)->Opcode()],
2255
                    NodeClassNames[argument(6)->Opcode()]);
2256
    return false; // not enough info for intrinsification
2257
  }
2258
  if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2259
    log_if_needed("  ** klass argument not initialized");
2260
    return false;
2261
  }
2262

2263
  assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST  ||
2264
         opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2265
         opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2266
  bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2267
  bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2268

2269
  ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2270
  ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2271
  if (is_vector_shuffle(vbox_klass_from)) {
2272
    return false; // vector shuffles aren't supported
2273
  }
2274
  bool is_mask = is_vector_mask(vbox_klass_from);
2275

2276
  ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2277
  if (!elem_type_from->is_primitive_type()) {
2278
    return false; // should be primitive type
2279
  }
2280
  BasicType elem_bt_from = elem_type_from->basic_type();
2281
  ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2282
  if (!elem_type_to->is_primitive_type()) {
2283
    return false; // should be primitive type
2284
  }
2285
  BasicType elem_bt_to = elem_type_to->basic_type();
2286

2287
  int num_elem_from = vlen_from->get_con();
2288
  int num_elem_to = vlen_to->get_con();
2289

2290
  // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2291
  // since we may need to change size.
2292
  if (!arch_supports_vector(Op_VectorReinterpret,
2293
                            num_elem_from,
2294
                            elem_bt_from,
2295
                            is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2296
    log_if_needed("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2297
                    is_cast ? "cast" : "reinterpret",
2298
                    num_elem_from, type2name(elem_bt_from), is_mask);
2299
    return false;
2300
  }
2301

2302
  // Check whether we can support resizing/reinterpreting to the new size.
2303
  if (!arch_supports_vector(Op_VectorReinterpret,
2304
                            num_elem_to,
2305
                            elem_bt_to,
2306
                            is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2307
    log_if_needed("  ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2308
                    is_cast ? "cast" : "reinterpret",
2309
                    num_elem_to, type2name(elem_bt_to), is_mask);
2310
    return false;
2311
  }
2312

2313

2314
  if (is_vector_shuffle(vbox_klass_to) &&
2315
      (!arch_supports_vector(Op_SubVB, num_elem_to, elem_bt_to, VecMaskNotUsed)           ||
2316
       !arch_supports_vector(Op_VectorBlend, num_elem_to, elem_bt_to, VecMaskNotUsed)     ||
2317
       !arch_supports_vector(Op_VectorMaskCmp, num_elem_to, elem_bt_to, VecMaskNotUsed)   ||
2318
       !arch_supports_vector(Op_AndV, num_elem_to, elem_bt_to, VecMaskNotUsed)            ||
2319
       !arch_supports_vector(Op_Replicate, num_elem_to, elem_bt_to, VecMaskNotUsed))) {
2320
    log_if_needed("  ** not supported: arity=1 op=shuffle_index_wrap vlen2=%d etype2=%s",
2321
                    num_elem_to, type2name(elem_bt_to));
2322
    return false;
2323
  }
2324

2325
  // At this point, we know that both input and output vector registers are supported
2326
  // by the architecture. Next check if the casted type is simply to same type - which means
2327
  // that it is actually a resize and not a cast.
2328
  if (is_cast && elem_bt_from == elem_bt_to) {
2329
    is_cast = false;
2330
  }
2331

2332
  const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2333

2334
  Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2335
  if (opd1 == nullptr) {
2336
    return false;
2337
  }
2338

2339
  const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2340
  const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2341

2342
  // Safety check to prevent casting if source mask is of type vector
2343
  // and destination mask of type predicate vector and vice-versa.
2344
  // From X86 standpoint, this case will only arise over KNL target,
2345
  // where certain masks (depending on the species) are either propagated
2346
  // through a vector or predicate register.
2347
  if (is_mask &&
2348
      ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2349
       (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2350
    return false;
2351
  }
2352

2353
  Node* op = opd1;
2354
  if (is_cast) {
2355
    assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2356
    int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2357

2358
    // Make sure that vector cast is implemented to particular type/size combination if it is
2359
    // not a mask casting.
2360
    if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2361
      log_if_needed("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2362
                      cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2363
      return false;
2364
    }
2365

2366
    if (num_elem_from < num_elem_to) {
2367
      // Since input and output number of elements are not consistent, we need to make sure we
2368
      // properly size. Thus, first make a cast that retains the number of elements from source.
2369
      int num_elem_for_cast = num_elem_from;
2370

2371
      // It is possible that arch does not support this intermediate vector size
2372
      // TODO More complex logic required here to handle this corner case for the sizes.
2373
      if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2374
        log_if_needed("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2375
                        cast_vopc,
2376
                        num_elem_for_cast, type2name(elem_bt_to), is_mask);
2377
        return false;
2378
      }
2379

2380
      op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2381
      // Now ensure that the destination gets properly resized to needed size.
2382
      op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2383
    } else if (num_elem_from > num_elem_to) {
2384
      // Since number of elements from input is larger than output, simply reduce size of input
2385
      // (we are supposed to drop top elements anyway).
2386
      int num_elem_for_resize = num_elem_to;
2387

2388
      // It is possible that arch does not support this intermediate vector size
2389
      // TODO More complex logic required here to handle this corner case for the sizes.
2390
      if (!arch_supports_vector(Op_VectorReinterpret,
2391
                                num_elem_for_resize,
2392
                                elem_bt_from,
2393
                                VecMaskNotUsed)) {
2394
        log_if_needed("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2395
                        num_elem_for_resize, type2name(elem_bt_from), is_mask);
2396
        return false;
2397
      }
2398

2399
      const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2400
      op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2401
      op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2402
    } else { // num_elem_from == num_elem_to
2403
      if (is_mask) {
2404
        // Make sure that cast for vector mask is implemented to particular type/size combination.
2405
        if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2406
          log_if_needed("  ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2407
                          num_elem_to, type2name(elem_bt_to), is_mask);
2408
          return false;
2409
        }
2410
        op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2411
      } else {
2412
        // Since input and output number of elements match, and since we know this vector size is
2413
        // supported, simply do a cast with no resize needed.
2414
        op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2415
      }
2416
    }
2417
  } else if (!Type::equals(src_type, dst_type)) {
2418
    assert(!is_cast, "must be reinterpret");
2419
    op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2420
  }
2421

2422
  if (is_vector_shuffle(vbox_klass_to)) {
2423
     op = partially_wrap_indexes(op, num_elem_to, elem_bt_to);
2424
  }
2425

2426
  const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2427
  Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2428
  set_result(vbox);
2429
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2430
  return true;
2431
}
2432

2433
//  public static
2434
//  <V extends Vector<E>,
2435
//   E>
2436
//  V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2437
//           V vec, int ix, long val,
2438
//           VecInsertOp<V> defaultImpl)
2439
bool LibraryCallKit::inline_vector_insert() {
2440
  const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2441
  const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2442
  const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2443
  const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2444

2445
  if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2446
    return false; // dead code
2447
  }
2448
  if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2449
    log_if_needed("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2450
                    NodeClassNames[argument(0)->Opcode()],
2451
                    NodeClassNames[argument(1)->Opcode()],
2452
                    NodeClassNames[argument(2)->Opcode()],
2453
                    NodeClassNames[argument(4)->Opcode()]);
2454
    return false; // not enough info for intrinsification
2455
  }
2456
  if (!is_klass_initialized(vector_klass)) {
2457
    log_if_needed("  ** klass argument not initialized");
2458
    return false;
2459
  }
2460
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2461
  if (!elem_type->is_primitive_type()) {
2462
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
2463
    return false; // should be primitive type
2464
  }
2465
  BasicType elem_bt = elem_type->basic_type();
2466
  int num_elem = vlen->get_con();
2467
  if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2468
    log_if_needed("  ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2469
                    num_elem, type2name(elem_bt));
2470
    return false; // not supported
2471
  }
2472

2473
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2474
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2475

2476
  Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2477
  if (opd == nullptr) {
2478
    return false;
2479
  }
2480

2481
  Node* insert_val = argument(5);
2482
  assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2483

2484
  // Convert insert value back to its appropriate type.
2485
  switch (elem_bt) {
2486
    case T_BYTE:
2487
      insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2488
      break;
2489
    case T_SHORT:
2490
      insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2491
      break;
2492
    case T_INT:
2493
      insert_val = gvn().transform(new ConvL2INode(insert_val));
2494
      break;
2495
    case T_FLOAT:
2496
      insert_val = gvn().transform(new ConvL2INode(insert_val));
2497
      insert_val = gvn().transform(new MoveI2FNode(insert_val));
2498
      break;
2499
    case T_DOUBLE:
2500
      insert_val = gvn().transform(new MoveL2DNode(insert_val));
2501
      break;
2502
    case T_LONG:
2503
      // no conversion needed
2504
      break;
2505
    default: fatal("%s", type2name(elem_bt)); break;
2506
  }
2507

2508
  Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2509

2510
  Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2511
  set_result(vbox);
2512
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2513
  return true;
2514
}
2515

2516
//  public static
2517
//  <VM extends VectorPayload,
2518
//   E>
2519
//  long extract(Class<? extends VM> vClass, Class<E> eClass,
2520
//               int length,
2521
//               VM vm, int i,
2522
//               VecExtractOp<VM> defaultImpl)
2523
bool LibraryCallKit::inline_vector_extract() {
2524
  const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2525
  const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2526
  const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2527
  const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2528

2529
  if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2530
    return false; // dead code
2531
  }
2532
  if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2533
    log_if_needed("  ** missing constant: vclass=%s etype=%s vlen=%s",
2534
                    NodeClassNames[argument(0)->Opcode()],
2535
                    NodeClassNames[argument(1)->Opcode()],
2536
                    NodeClassNames[argument(2)->Opcode()]);
2537
    return false; // not enough info for intrinsification
2538
  }
2539
  if (!is_klass_initialized(vector_klass)) {
2540
    log_if_needed("  ** klass argument not initialized");
2541
    return false;
2542
  }
2543
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2544
  if (!elem_type->is_primitive_type()) {
2545
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
2546
    return false; // should be primitive type
2547
  }
2548
  BasicType elem_bt = elem_type->basic_type();
2549
  int num_elem = vlen->get_con();
2550

2551
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2552
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2553

2554
  Node* opd = nullptr;
2555

2556
  if (is_vector_mask(vbox_klass)) {
2557
    // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2558

2559
    Node* pos = argument(4); // can be variable
2560
    if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2561
      // Transform mask to vector with type of boolean and utilize ExtractUB node.
2562
      opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2563
      if (opd == nullptr) {
2564
        return false;
2565
      }
2566
      opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2567
      opd = gvn().transform(new ExtractUBNode(opd, pos));
2568
      opd = gvn().transform(new ConvI2LNode(opd));
2569
    } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2570
      opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2571
      if (opd == nullptr) {
2572
        return false;
2573
      }
2574
      // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2575
      if (opd->bottom_type()->isa_vectmask() == nullptr) {
2576
        opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2577
      }
2578
      // ((toLong() >>> pos) & 1L
2579
      opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2580
      opd = gvn().transform(new URShiftLNode(opd, pos));
2581
      opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2582
    } else {
2583
      log_if_needed("  ** Rejected mask extraction because architecture does not support it");
2584
      return false; // not supported
2585
    }
2586
  } else {
2587
    // vbox_klass is vector. This is used for Vector.lane(int).
2588
    if (!idx->is_con()) {
2589
      log_if_needed("  ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2590
      return false; // not enough info for intrinsification
2591
    }
2592

2593
    int vopc = ExtractNode::opcode(elem_bt);
2594
    if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2595
      log_if_needed("  ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2596
                      num_elem, type2name(elem_bt));
2597
      return false; // not supported
2598
    }
2599

2600
    opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2601
    if (opd == nullptr) {
2602
      return false;
2603
    }
2604
    ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2605

2606
    opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2607
    switch (elem_bt) {
2608
      case T_BYTE:
2609
      case T_SHORT:
2610
      case T_INT: {
2611
        opd = gvn().transform(new ConvI2LNode(opd));
2612
        break;
2613
      }
2614
      case T_FLOAT: {
2615
        opd = gvn().transform(new MoveF2INode(opd));
2616
        opd = gvn().transform(new ConvI2LNode(opd));
2617
        break;
2618
      }
2619
      case T_DOUBLE: {
2620
        opd = gvn().transform(new MoveD2LNode(opd));
2621
        break;
2622
      }
2623
      case T_LONG: {
2624
        // no conversion needed
2625
        break;
2626
      }
2627
      default: fatal("%s", type2name(elem_bt));
2628
    }
2629
  }
2630
  set_result(opd);
2631
  return true;
2632
}
2633

2634
// public static
2635
// <V extends Vector<E>,
2636
//  M extends VectorMask<E>,
2637
//  E>
2638
//  V compressExpandOp(int opr,
2639
//                    Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2640
//                    int length, V v, M m,
2641
//                    CompressExpandOperation<V, M> defaultImpl)
2642
bool LibraryCallKit::inline_vector_compress_expand() {
2643
  const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2644
  const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2645
  const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2646
  const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2647
  const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2648

2649
  if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2650
      vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2651
      elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2652
    log_if_needed("  ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2653
                    NodeClassNames[argument(0)->Opcode()],
2654
                    NodeClassNames[argument(1)->Opcode()],
2655
                    NodeClassNames[argument(2)->Opcode()],
2656
                    NodeClassNames[argument(3)->Opcode()],
2657
                    NodeClassNames[argument(4)->Opcode()]);
2658
    return false; // not enough info for intrinsification
2659
  }
2660

2661
  if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2662
    log_if_needed("  ** klass argument not initialized");
2663
    return false;
2664
  }
2665

2666
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2667
  if (!elem_type->is_primitive_type()) {
2668
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
2669
    return false; // should be primitive type
2670
  }
2671

2672
  int num_elem = vlen->get_con();
2673
  BasicType elem_bt = elem_type->basic_type();
2674
  int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2675

2676
  if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2677
    log_if_needed("  ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2678
                    opc, num_elem, type2name(elem_bt));
2679
    return false; // not supported
2680
  }
2681

2682
  Node* opd1 = nullptr;
2683
  const TypeInstPtr* vbox_type = nullptr;
2684
  if (opc != Op_CompressM) {
2685
    ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2686
    vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2687
    opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2688
    if (opd1 == nullptr) {
2689
      log_if_needed("  ** unbox failed vector=%s",
2690
                      NodeClassNames[argument(5)->Opcode()]);
2691
      return false;
2692
    }
2693
  }
2694

2695
  ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2696
  assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2697
  const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2698

2699
  Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2700
  if (mask == nullptr) {
2701
    log_if_needed("  ** unbox failed mask=%s",
2702
                    NodeClassNames[argument(6)->Opcode()]);
2703
    return false;
2704
  }
2705

2706
  const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2707
  Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2708

2709
  // Wrap it up in VectorBox to keep object type information.
2710
  const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2711
  Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2712
  set_result(vbox);
2713
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2714
  return true;
2715
}
2716

2717
// public static
2718
// <V extends Vector<E>,
2719
//  E,
2720
//  S extends VectorSpecies<E>>
2721
//  V indexVector(Class<? extends V> vClass, Class<E> eClass,
2722
//                int length,
2723
//                V v, int step, S s,
2724
//                IndexOperation<V, S> defaultImpl)
2725
bool LibraryCallKit::inline_index_vector() {
2726
  const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2727
  const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2728
  const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2729

2730
  if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2731
      vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2732
      elem_klass->const_oop() == nullptr) {
2733
    log_if_needed("  ** missing constant: vclass=%s etype=%s vlen=%s",
2734
                    NodeClassNames[argument(0)->Opcode()],
2735
                    NodeClassNames[argument(1)->Opcode()],
2736
                    NodeClassNames[argument(2)->Opcode()]);
2737
    return false; // not enough info for intrinsification
2738
  }
2739

2740
  if (!is_klass_initialized(vector_klass)) {
2741
    log_if_needed("  ** klass argument not initialized");
2742
    return false;
2743
  }
2744

2745
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2746
  if (!elem_type->is_primitive_type()) {
2747
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
2748
    return false; // should be primitive type
2749
  }
2750

2751
  int num_elem = vlen->get_con();
2752
  BasicType elem_bt = elem_type->basic_type();
2753

2754
  // Check whether the iota index generation op is supported by the current hardware
2755
  if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2756
    log_if_needed("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2757
    return false; // not supported
2758
  }
2759

2760
  int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2761
  int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2762
  bool needs_mul = true;
2763
  Node* scale = argument(4);
2764
  const TypeInt* scale_type = gvn().type(scale)->isa_int();
2765
  // Multiply is not needed if the scale is a constant "1".
2766
  if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2767
    needs_mul = false;
2768
  } else {
2769
    // Check whether the vector multiply op is supported by the current hardware
2770
    if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2771
      log_if_needed("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2772
      return false; // not supported
2773
    }
2774

2775
    // Check whether the scalar cast op is supported by the current hardware
2776
    if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2777
      int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2778
                    elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2779
      if (!Matcher::match_rule_supported(cast_op)) {
2780
        log_if_needed("  ** Rejected op (%s) because architecture does not support it",
2781
                        NodeClassNames[cast_op]);
2782
        return false; // not supported
2783
      }
2784
    }
2785
  }
2786

2787
  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2788
  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2789
  Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2790
  if (opd == nullptr) {
2791
    log_if_needed("  ** unbox failed vector=%s",
2792
                    NodeClassNames[argument(3)->Opcode()]);
2793
    return false;
2794
  }
2795

2796
  int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2797
  int vadd_op = VectorNode::opcode(add_op, elem_bt);
2798
  bool needs_add = true;
2799
  // The addition is not needed if all the element values of "opd" are zero
2800
  if (VectorNode::is_all_zeros_vector(opd)) {
2801
    needs_add = false;
2802
  } else {
2803
    // Check whether the vector addition op is supported by the current hardware
2804
    if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2805
      log_if_needed("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2806
      return false; // not supported
2807
    }
2808
  }
2809

2810
  // Compute the iota indice vector
2811
  const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2812
  Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2813

2814
  // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2815
  if (needs_mul) {
2816
    switch (elem_bt) {
2817
      case T_BOOLEAN: // fall-through
2818
      case T_BYTE:    // fall-through
2819
      case T_SHORT:   // fall-through
2820
      case T_CHAR:    // fall-through
2821
      case T_INT: {
2822
        // no conversion needed
2823
        break;
2824
      }
2825
      case T_LONG: {
2826
        scale = gvn().transform(new ConvI2LNode(scale));
2827
        break;
2828
      }
2829
      case T_FLOAT: {
2830
        scale = gvn().transform(new ConvI2FNode(scale));
2831
        break;
2832
      }
2833
      case T_DOUBLE: {
2834
        scale = gvn().transform(new ConvI2DNode(scale));
2835
        break;
2836
      }
2837
      default: fatal("%s", type2name(elem_bt));
2838
    }
2839
    scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, Type::get_const_basic_type(elem_bt)));
2840
    index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
2841
  }
2842

2843
  // Add "opd" if addition is needed.
2844
  if (needs_add) {
2845
    index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
2846
  }
2847
  Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
2848
  set_result(vbox);
2849
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2850
  return true;
2851
}
2852

2853
// public static
2854
// <E,
2855
//  M extends VectorMask<E>>
2856
// M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
2857
//                              long offset, long limit,
2858
//                              IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
2859
bool LibraryCallKit::inline_index_partially_in_upper_range() {
2860
  const TypeInstPtr* mask_klass   = gvn().type(argument(0))->isa_instptr();
2861
  const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2862
  const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2863

2864
  if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2865
      mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2866
    log_if_needed("  ** missing constant: mclass=%s etype=%s vlen=%s",
2867
                    NodeClassNames[argument(0)->Opcode()],
2868
                    NodeClassNames[argument(1)->Opcode()],
2869
                    NodeClassNames[argument(2)->Opcode()]);
2870
    return false; // not enough info for intrinsification
2871
  }
2872

2873
  if (!is_klass_initialized(mask_klass)) {
2874
    log_if_needed("  ** klass argument not initialized");
2875
    return false;
2876
  }
2877

2878
  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2879
  if (!elem_type->is_primitive_type()) {
2880
    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
2881
    return false; // should be primitive type
2882
  }
2883

2884
  int num_elem = vlen->get_con();
2885
  BasicType elem_bt = elem_type->basic_type();
2886

2887
  // Check whether the necessary ops are supported by current hardware.
2888
  bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
2889
  if (!supports_mask_gen) {
2890
    if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
2891
        !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
2892
        !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
2893
      log_if_needed("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2894
      return false; // not supported
2895
    }
2896

2897
    // Check whether the scalar cast operation is supported by current hardware.
2898
    if (elem_bt != T_LONG) {
2899
      int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
2900
                                              : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
2901
      if (!Matcher::match_rule_supported(cast_op)) {
2902
        log_if_needed("  ** Rejected op (%s) because architecture does not support it",
2903
                        NodeClassNames[cast_op]);
2904
        return false; // not supported
2905
      }
2906
    }
2907
  }
2908

2909
  Node* offset = argument(3);
2910
  Node* limit = argument(5);
2911
  if (offset == nullptr || limit == nullptr) {
2912
    log_if_needed("  ** offset or limit argument is null");
2913
    return false; // not supported
2914
  }
2915

2916
  ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2917
  assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
2918
  const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
2919

2920
  // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
2921
  // So directly get indexLimit with "indexLimit = limit - offset".
2922
  Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
2923
  Node* mask = nullptr;
2924
  if (supports_mask_gen) {
2925
    mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
2926
  } else {
2927
    // Generate the vector mask based on "mask = iota < indexLimit".
2928
    // Broadcast "indexLimit" to a vector.
2929
    switch (elem_bt) {
2930
      case T_BOOLEAN: // fall-through
2931
      case T_BYTE:    // fall-through
2932
      case T_SHORT:   // fall-through
2933
      case T_CHAR:    // fall-through
2934
      case T_INT: {
2935
        indexLimit = gvn().transform(new ConvL2INode(indexLimit));
2936
        break;
2937
      }
2938
      case T_DOUBLE: {
2939
        indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
2940
        break;
2941
      }
2942
      case T_FLOAT: {
2943
        indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
2944
        break;
2945
      }
2946
      case T_LONG: {
2947
        // no conversion needed
2948
        break;
2949
      }
2950
      default: fatal("%s", type2name(elem_bt));
2951
    }
2952
    indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, Type::get_const_basic_type(elem_bt)));
2953

2954
    // Load the "iota" vector.
2955
    const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2956
    Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2957

2958
    // Compute the vector mask with "mask = iota < indexLimit".
2959
    ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
2960
    const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
2961
    mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
2962
  }
2963
  Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
2964
  set_result(vbox);
2965
  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2966
  return true;
2967
}
2968

2969
#undef non_product_log_if_needed
2970
#undef log_if_needed
2971