jdk

1
/*
2
 * Copyright (c) 2007, 2024, Oracle and/or its affiliates. All rights reserved.
3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4
 *
5
 * This code is free software; you can redistribute it and/or modify it
6
 * under the terms of the GNU General Public License version 2 only, as
7
 * published by the Free Software Foundation.
8
 *
9
 * This code is distributed in the hope that it will be useful, but WITHOUT
10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12
 * version 2 for more details (a copy is included in the LICENSE file that
13
 * accompanied this code).
14
 *
15
 * You should have received a copy of the GNU General Public License version
16
 * 2 along with this work; if not, write to the Free Software Foundation,
17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18
 *
19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20
 * or visit www.oracle.com if you need additional information or have any
21
 * questions.
22
 */
23

24
#include "precompiled.hpp"
25
#include "memory/allocation.inline.hpp"
26
#include "opto/connode.hpp"
27
#include "opto/mulnode.hpp"
28
#include "opto/subnode.hpp"
29
#include "opto/vectornode.hpp"
30
#include "opto/convertnode.hpp"
31
#include "utilities/powerOfTwo.hpp"
32
#include "utilities/globalDefinitions.hpp"
33

34
//------------------------------VectorNode--------------------------------------
35

36
// Return the vector operator for the specified scalar operation
37
// and basic type.
38
int VectorNode::opcode(int sopc, BasicType bt) {
39
  switch (sopc) {
40
  case Op_AddI:
41
    switch (bt) {
42
    case T_BOOLEAN:
43
    case T_BYTE:      return Op_AddVB;
44
    case T_CHAR:
45
    case T_SHORT:     return Op_AddVS;
46
    case T_INT:       return Op_AddVI;
47
    default:          return 0;
48
    }
49
  case Op_AddL: return (bt == T_LONG   ? Op_AddVL : 0);
50
  case Op_AddF: return (bt == T_FLOAT  ? Op_AddVF : 0);
51
  case Op_AddD: return (bt == T_DOUBLE ? Op_AddVD : 0);
52

53
  case Op_SubI:
54
    switch (bt) {
55
    case T_BOOLEAN:
56
    case T_BYTE:   return Op_SubVB;
57
    case T_CHAR:
58
    case T_SHORT:  return Op_SubVS;
59
    case T_INT:    return Op_SubVI;
60
    default:       return 0;
61
    }
62
  case Op_SubL: return (bt == T_LONG   ? Op_SubVL : 0);
63
  case Op_SubF: return (bt == T_FLOAT  ? Op_SubVF : 0);
64
  case Op_SubD: return (bt == T_DOUBLE ? Op_SubVD : 0);
65

66
  case Op_MulI:
67
    switch (bt) {
68
    case T_BOOLEAN:return 0;
69
    case T_BYTE:   return Op_MulVB;
70
    case T_CHAR:
71
    case T_SHORT:  return Op_MulVS;
72
    case T_INT:    return Op_MulVI;
73
    default:       return 0;
74
    }
75
  case Op_MulL: return (bt == T_LONG ? Op_MulVL : 0);
76
  case Op_MulF:
77
    return (bt == T_FLOAT ? Op_MulVF : 0);
78
  case Op_MulD:
79
    return (bt == T_DOUBLE ? Op_MulVD : 0);
80
  case Op_FmaD:
81
    return (bt == T_DOUBLE ? Op_FmaVD : 0);
82
  case Op_FmaF:
83
    return (bt == T_FLOAT ? Op_FmaVF : 0);
84
  case Op_CMoveF:
85
    return (bt == T_FLOAT ? Op_VectorBlend : 0);
86
  case Op_CMoveD:
87
    return (bt == T_DOUBLE ? Op_VectorBlend : 0);
88
  case Op_Bool:
89
    return Op_VectorMaskCmp;
90
  case Op_DivF:
91
    return (bt == T_FLOAT ? Op_DivVF : 0);
92
  case Op_DivD:
93
    return (bt == T_DOUBLE ? Op_DivVD : 0);
94
  case Op_AbsI:
95
    switch (bt) {
96
    case T_BOOLEAN:
97
    case T_CHAR:  return 0; // abs does not make sense for unsigned
98
    case T_BYTE:  return Op_AbsVB;
99
    case T_SHORT: return Op_AbsVS;
100
    case T_INT:   return Op_AbsVI;
101
    default:      return 0;
102
    }
103
  case Op_AbsL:
104
    return (bt == T_LONG ? Op_AbsVL : 0);
105
  case Op_MinI:
106
    switch (bt) {
107
    case T_BOOLEAN:
108
    case T_CHAR:   return 0;
109
    case T_BYTE:
110
    case T_SHORT:
111
    case T_INT:    return Op_MinV;
112
    default:       return 0;
113
    }
114
  case Op_MinL:
115
    return (bt == T_LONG ? Op_MinV : 0);
116
  case Op_MinF:
117
    return (bt == T_FLOAT ? Op_MinV : 0);
118
  case Op_MinD:
119
    return (bt == T_DOUBLE ? Op_MinV : 0);
120
  case Op_MaxI:
121
    switch (bt) {
122
    case T_BOOLEAN:
123
    case T_CHAR:   return 0;
124
    case T_BYTE:
125
    case T_SHORT:
126
    case T_INT:    return Op_MaxV;
127
    default:       return 0;
128
    }
129
  case Op_MaxL:
130
    return (bt == T_LONG ? Op_MaxV : 0);
131
  case Op_MaxF:
132
    return (bt == T_FLOAT ? Op_MaxV : 0);
133
  case Op_MaxD:
134
    return (bt == T_DOUBLE ? Op_MaxV : 0);
135
  case Op_AbsF:
136
    return (bt == T_FLOAT ? Op_AbsVF : 0);
137
  case Op_AbsD:
138
    return (bt == T_DOUBLE ? Op_AbsVD : 0);
139
  case Op_NegI:
140
    switch (bt) {
141
      case T_BYTE:
142
      case T_SHORT:
143
      case T_INT: return Op_NegVI;
144
      default: return 0;
145
    }
146
  case Op_NegL:
147
    return (bt == T_LONG ? Op_NegVL : 0);
148
  case Op_NegF:
149
    return (bt == T_FLOAT ? Op_NegVF : 0);
150
  case Op_NegD:
151
    return (bt == T_DOUBLE ? Op_NegVD : 0);
152
  case Op_RoundDoubleMode:
153
    return (bt == T_DOUBLE ? Op_RoundDoubleModeV : 0);
154
  case Op_RotateLeft:
155
    return (is_integral_type(bt) ? Op_RotateLeftV : 0);
156
  case Op_RotateRight:
157
    return (is_integral_type(bt) ? Op_RotateRightV : 0);
158
  case Op_SqrtF:
159
    return (bt == T_FLOAT ? Op_SqrtVF : 0);
160
  case Op_SqrtD:
161
    return (bt == T_DOUBLE ? Op_SqrtVD : 0);
162
  case Op_RoundF:
163
    return (bt == T_INT ? Op_RoundVF : 0);
164
  case Op_RoundD:
165
    return (bt == T_LONG ? Op_RoundVD : 0);
166
  case Op_PopCountI:
167
    return Op_PopCountVI;
168
  case Op_PopCountL:
169
    return Op_PopCountVL;
170
  case Op_ReverseI:
171
  case Op_ReverseL:
172
    return (is_integral_type(bt) ? Op_ReverseV : 0);
173
  case Op_ReverseBytesS:
174
  case Op_ReverseBytesUS:
175
    // Subword operations in auto vectorization usually don't have precise info
176
    // about signedness. But the behavior of reverseBytes for short and
177
    // char are exactly the same.
178
    return ((bt == T_SHORT || bt == T_CHAR) ? Op_ReverseBytesV : 0);
179
  case Op_ReverseBytesI:
180
    // There is no reverseBytes() in Byte class but T_BYTE may appear
181
    // in VectorAPI calls. We still use ReverseBytesI for T_BYTE to
182
    // ensure vector intrinsification succeeds.
183
    return ((bt == T_INT || bt == T_BYTE) ? Op_ReverseBytesV : 0);
184
  case Op_ReverseBytesL:
185
    return (bt == T_LONG ? Op_ReverseBytesV : 0);
186
  case Op_CompressBits:
187
    return (bt == T_INT || bt == T_LONG ? Op_CompressBitsV : 0);
188
  case Op_ExpandBits:
189
    return (bt == T_INT || bt == T_LONG ? Op_ExpandBitsV : 0);
190
  case Op_LShiftI:
191
    switch (bt) {
192
    case T_BOOLEAN:
193
    case T_BYTE:   return Op_LShiftVB;
194
    case T_CHAR:
195
    case T_SHORT:  return Op_LShiftVS;
196
    case T_INT:    return Op_LShiftVI;
197
    default:       return 0;
198
    }
199
  case Op_LShiftL:
200
    return (bt == T_LONG ? Op_LShiftVL : 0);
201
  case Op_RShiftI:
202
    switch (bt) {
203
    case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value
204
    case T_CHAR:   return Op_URShiftVS; // char is unsigned value
205
    case T_BYTE:   return Op_RShiftVB;
206
    case T_SHORT:  return Op_RShiftVS;
207
    case T_INT:    return Op_RShiftVI;
208
    default:       return 0;
209
    }
210
  case Op_RShiftL:
211
    return (bt == T_LONG ? Op_RShiftVL : 0);
212
  case Op_URShiftB:
213
    return (bt == T_BYTE ? Op_URShiftVB : 0);
214
  case Op_URShiftS:
215
    return (bt == T_SHORT ? Op_URShiftVS : 0);
216
  case Op_URShiftI:
217
    switch (bt) {
218
    case T_BOOLEAN:return Op_URShiftVB;
219
    case T_CHAR:   return Op_URShiftVS;
220
    case T_BYTE:
221
    case T_SHORT:  return 0; // Vector logical right shift for signed short
222
                             // values produces incorrect Java result for
223
                             // negative data because java code should convert
224
                             // a short value into int value with sign
225
                             // extension before a shift.
226
    case T_INT:    return Op_URShiftVI;
227
    default:       return 0;
228
    }
229
  case Op_URShiftL:
230
    return (bt == T_LONG ? Op_URShiftVL : 0);
231
  case Op_AndI:
232
  case Op_AndL:
233
    return Op_AndV;
234
  case Op_OrI:
235
  case Op_OrL:
236
    return Op_OrV;
237
  case Op_XorI:
238
  case Op_XorL:
239
    return Op_XorV;
240

241
  case Op_LoadB:
242
  case Op_LoadUB:
243
  case Op_LoadUS:
244
  case Op_LoadS:
245
  case Op_LoadI:
246
  case Op_LoadL:
247
  case Op_LoadF:
248
  case Op_LoadD:
249
    return Op_LoadVector;
250

251
  case Op_StoreB:
252
  case Op_StoreC:
253
  case Op_StoreI:
254
  case Op_StoreL:
255
  case Op_StoreF:
256
  case Op_StoreD:
257
    return Op_StoreVector;
258
  case Op_MulAddS2I:
259
    return Op_MulAddVS2VI;
260
  case Op_CountLeadingZerosI:
261
  case Op_CountLeadingZerosL:
262
    return Op_CountLeadingZerosV;
263
  case Op_CountTrailingZerosI:
264
  case Op_CountTrailingZerosL:
265
    return Op_CountTrailingZerosV;
266
  case Op_SignumF:
267
    return Op_SignumVF;
268
  case Op_SignumD:
269
    return Op_SignumVD;
270

271
  default:
272
    assert(!VectorNode::is_convert_opcode(sopc),
273
           "Convert node %s should be processed by VectorCastNode::opcode()",
274
           NodeClassNames[sopc]);
275
    return 0; // Unimplemented
276
  }
277
}
278

279
// Return the scalar opcode for the specified vector opcode
280
// and basic type.
281
int VectorNode::scalar_opcode(int sopc, BasicType bt) {
282
  switch (sopc) {
283
    case Op_AddReductionVI:
284
    case Op_AddVI:
285
      return Op_AddI;
286
    case Op_AddReductionVL:
287
    case Op_AddVL:
288
      return Op_AddL;
289
    case Op_MulReductionVI:
290
    case Op_MulVI:
291
      return Op_MulI;
292
    case Op_MulReductionVL:
293
    case Op_MulVL:
294
      return Op_MulL;
295
    case Op_AndReductionV:
296
    case Op_AndV:
297
      switch (bt) {
298
        case T_BOOLEAN:
299
        case T_CHAR:
300
        case T_BYTE:
301
        case T_SHORT:
302
        case T_INT:
303
          return Op_AndI;
304
        case T_LONG:
305
          return Op_AndL;
306
        default:
307
          assert(false, "basic type not handled");
308
          return 0;
309
      }
310
    case Op_OrReductionV:
311
    case Op_OrV:
312
      switch (bt) {
313
        case T_BOOLEAN:
314
        case T_CHAR:
315
        case T_BYTE:
316
        case T_SHORT:
317
        case T_INT:
318
          return Op_OrI;
319
        case T_LONG:
320
          return Op_OrL;
321
        default:
322
          assert(false, "basic type not handled");
323
          return 0;
324
      }
325
    case Op_XorReductionV:
326
    case Op_XorV:
327
      switch (bt) {
328
        case T_BOOLEAN:
329
        case T_CHAR:
330
        case T_BYTE:
331
        case T_SHORT:
332
        case T_INT:
333
          return Op_XorI;
334
        case T_LONG:
335
          return Op_XorL;
336
        default:
337
          assert(false, "basic type not handled");
338
          return 0;
339
      }
340
    case Op_MinReductionV:
341
    case Op_MinV:
342
      switch (bt) {
343
        case T_BOOLEAN:
344
        case T_CHAR:
345
          assert(false, "boolean and char are signed, not implemented for Min");
346
          return 0;
347
        case T_BYTE:
348
        case T_SHORT:
349
        case T_INT:
350
          return Op_MinI;
351
        case T_LONG:
352
          return Op_MinL;
353
        case T_FLOAT:
354
          return Op_MinF;
355
        case T_DOUBLE:
356
          return Op_MinD;
357
        default:
358
          assert(false, "basic type not handled");
359
          return 0;
360
      }
361
    case Op_MaxReductionV:
362
    case Op_MaxV:
363
      switch (bt) {
364
        case T_BOOLEAN:
365
        case T_CHAR:
366
          assert(false, "boolean and char are signed, not implemented for Max");
367
          return 0;
368
        case T_BYTE:
369
        case T_SHORT:
370
        case T_INT:
371
          return Op_MaxI;
372
        case T_LONG:
373
          return Op_MaxL;
374
        case T_FLOAT:
375
          return Op_MaxF;
376
        case T_DOUBLE:
377
          return Op_MaxD;
378
        default:
379
          assert(false, "basic type not handled");
380
          return 0;
381
      }
382
    default:
383
      assert(false,
384
             "Vector node %s is not handled in VectorNode::scalar_opcode",
385
             NodeClassNames[sopc]);
386
      return 0; // Unimplemented
387
  }
388
}
389

390
// Limits on vector size (number of elements) for auto-vectorization.
391
bool VectorNode::vector_size_supported_auto_vectorization(const BasicType bt, int size) {
392
  return Matcher::max_vector_size_auto_vectorization(bt) >= size &&
393
         Matcher::min_vector_size(bt) <= size;
394
}
395

396
// Also used to check if the code generator
397
// supports the vector operation.
398
bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
399
  if (is_java_primitive(bt) &&
400
      (vlen > 1) && is_power_of_2(vlen) &&
401
      vector_size_supported_auto_vectorization(bt, vlen)) {
402
    int vopc = VectorNode::opcode(opc, bt);
403
    // For rotate operation we will do a lazy de-generation into
404
    // OrV/LShiftV/URShiftV pattern if the target does not support
405
    // vector rotation instruction.
406
    if (VectorNode::is_vector_rotate(vopc)) {
407
      return is_vector_rotate_supported(vopc, vlen, bt);
408
    }
409
    if (VectorNode::is_vector_integral_negate(vopc)) {
410
      return is_vector_integral_negate_supported(vopc, vlen, bt, false);
411
    }
412
    return vopc > 0 && Matcher::match_rule_supported_auto_vectorization(vopc, vlen, bt);
413
  }
414
  return false;
415
}
416

417
bool VectorNode::is_muladds2i(const Node* n) {
418
  return n->Opcode() == Op_MulAddS2I;
419
}
420

421
bool VectorNode::is_roundopD(Node* n) {
422
  return n->Opcode() == Op_RoundDoubleMode;
423
}
424

425
bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
426
  assert(VectorNode::is_vector_rotate(vopc), "wrong opcode");
427

428
  // If target defines vector rotation patterns then no
429
  // need for degeneration.
430
  if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) {
431
    return true;
432
  }
433

434
  // If target does not support variable shift operations then no point
435
  // in creating a rotate vector node since it will not be disintegratable.
436
  // Adding a pessimistic check to avoid complex pattern matching which
437
  // may not be full proof.
438
  if (!Matcher::supports_vector_variable_shifts()) {
439
     return false;
440
  }
441

442
  // Validate existence of nodes created in case of rotate degeneration.
443
  switch (bt) {
444
    case T_INT:
445
      return Matcher::match_rule_supported_vector(Op_OrV,       vlen, bt) &&
446
             Matcher::match_rule_supported_vector(Op_LShiftVI,  vlen, bt) &&
447
             Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt);
448
    case T_LONG:
449
      return Matcher::match_rule_supported_vector(Op_OrV,       vlen, bt) &&
450
             Matcher::match_rule_supported_vector(Op_LShiftVL,  vlen, bt) &&
451
             Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt);
452
    default:
453
      return false;
454
  }
455
}
456

457
// Check whether the architecture supports the vector negate instructions. If not, then check
458
// whether the alternative vector nodes used to implement vector negation are supported.
459
// Return false if neither of them is supported.
460
bool VectorNode::is_vector_integral_negate_supported(int opc, uint vlen, BasicType bt, bool use_predicate) {
461
  if (!use_predicate) {
462
    // Check whether the NegVI/L is supported by the architecture.
463
    if (Matcher::match_rule_supported_vector(opc, vlen, bt)) {
464
      return true;
465
    }
466
    // Negate is implemented with "(SubVI/L (ReplicateI/L 0) src)", if NegVI/L is not supported.
467
    int sub_opc = (bt == T_LONG) ? Op_SubL : Op_SubI;
468
    if (Matcher::match_rule_supported_vector(VectorNode::opcode(sub_opc, bt), vlen, bt) &&
469
        Matcher::match_rule_supported_vector(Op_Replicate, vlen, bt)) {
470
      return true;
471
    }
472
  } else {
473
    // Check whether the predicated NegVI/L is supported by the architecture.
474
    if (Matcher::match_rule_supported_vector_masked(opc, vlen, bt)) {
475
      return true;
476
    }
477
    // Predicated negate is implemented with "(AddVI/L (XorV src (ReplicateI/L -1)) (ReplicateI/L 1))",
478
    // if predicated NegVI/L is not supported.
479
    int add_opc = (bt == T_LONG) ? Op_AddL : Op_AddI;
480
    if (Matcher::match_rule_supported_vector_masked(Op_XorV, vlen, bt) &&
481
        Matcher::match_rule_supported_vector_masked(VectorNode::opcode(add_opc, bt), vlen, bt) &&
482
        Matcher::match_rule_supported_vector(Op_Replicate, vlen, bt)) {
483
      return true;
484
    }
485
  }
486
  return false;
487
}
488

489
bool VectorNode::is_populate_index_supported(BasicType bt) {
490
  int vlen = Matcher::max_vector_size(bt);
491
  return Matcher::match_rule_supported_vector(Op_PopulateIndex, vlen, bt);
492
}
493

494
bool VectorNode::is_shift_opcode(int opc) {
495
  switch (opc) {
496
  case Op_LShiftI:
497
  case Op_LShiftL:
498
  case Op_RShiftI:
499
  case Op_RShiftL:
500
  case Op_URShiftB:
501
  case Op_URShiftS:
502
  case Op_URShiftI:
503
  case Op_URShiftL:
504
    return true;
505
  default:
506
    return false;
507
  }
508
}
509

510
// Vector unsigned right shift for signed subword types behaves differently
511
// from Java Spec. But when the shift amount is a constant not greater than
512
// the number of sign extended bits, the unsigned right shift can be
513
// vectorized to a signed right shift.
514
bool VectorNode::can_use_RShiftI_instead_of_URShiftI(Node* n, BasicType bt) {
515
  if (n->Opcode() != Op_URShiftI) {
516
    return false;
517
  }
518
  Node* in2 = n->in(2);
519
  if (!in2->is_Con()) {
520
    return false;
521
  }
522
  jint cnt = in2->get_int();
523
  // Only when shift amount is not greater than number of sign extended
524
  // bits (16 for short and 24 for byte), unsigned shift right on signed
525
  // subword types can be vectorized as vector signed shift.
526
  if ((bt == T_BYTE && cnt <= 24) || (bt == T_SHORT && cnt <= 16)) {
527
    return true;
528
  }
529
  return false;
530
}
531

532
bool VectorNode::is_convert_opcode(int opc) {
533
  switch (opc) {
534
    case Op_ConvI2F:
535
    case Op_ConvL2D:
536
    case Op_ConvF2I:
537
    case Op_ConvD2L:
538
    case Op_ConvI2D:
539
    case Op_ConvL2F:
540
    case Op_ConvL2I:
541
    case Op_ConvI2L:
542
    case Op_ConvF2L:
543
    case Op_ConvD2F:
544
    case Op_ConvF2D:
545
    case Op_ConvD2I:
546
    case Op_ConvF2HF:
547
    case Op_ConvHF2F:
548
      return true;
549
    default:
550
      return false;
551
  }
552
}
553

554
bool VectorNode::is_minmax_opcode(int opc) {
555
  return opc == Op_MinI || opc == Op_MaxI;
556
}
557

558
bool VectorNode::is_shift(Node* n) {
559
  return is_shift_opcode(n->Opcode());
560
}
561

562
bool VectorNode::is_rotate_opcode(int opc) {
563
  switch (opc) {
564
  case Op_RotateRight:
565
  case Op_RotateLeft:
566
    return true;
567
  default:
568
    return false;
569
  }
570
}
571

572
bool VectorNode::is_scalar_rotate(Node* n) {
573
  return is_rotate_opcode(n->Opcode());
574
}
575

576
bool VectorNode::is_vshift_cnt_opcode(int opc) {
577
  switch (opc) {
578
  case Op_LShiftCntV:
579
  case Op_RShiftCntV:
580
    return true;
581
  default:
582
    return false;
583
  }
584
}
585

586
bool VectorNode::is_vshift_cnt(Node* n) {
587
  return is_vshift_cnt_opcode(n->Opcode());
588
}
589

590
// [Start, end) half-open range defining which operands are vectors
591
void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
592
  switch (n->Opcode()) {
593
  case Op_LoadB:   case Op_LoadUB:
594
  case Op_LoadS:   case Op_LoadUS:
595
  case Op_LoadI:   case Op_LoadL:
596
  case Op_LoadF:   case Op_LoadD:
597
  case Op_LoadP:   case Op_LoadN:
598
    *start = 0;
599
    *end   = 0; // no vector operands
600
    break;
601
  case Op_StoreB:  case Op_StoreC:
602
  case Op_StoreI:  case Op_StoreL:
603
  case Op_StoreF:  case Op_StoreD:
604
  case Op_StoreP:  case Op_StoreN:
605
    *start = MemNode::ValueIn;
606
    *end   = MemNode::ValueIn + 1; // 1 vector operand
607
    break;
608
  case Op_LShiftI:  case Op_LShiftL:
609
  case Op_RShiftI:  case Op_RShiftL:
610
  case Op_URShiftI: case Op_URShiftL:
611
  case Op_RoundDoubleMode:
612
    *start = 1;
613
    *end   = 2; // 1 vector operand
614
    break;
615
  case Op_RotateLeft:
616
  case Op_RotateRight:
617
    // Rotate shift could have 1 or 2 vector operand(s), depending on
618
    // whether shift distance is a supported constant or not.
619
    *start = 1;
620
    *end   = (n->is_Con() && Matcher::supports_vector_constant_rotates(n->get_int())) ? 2 : 3;
621
    break;
622
  case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD:
623
  case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD:
624
  case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD:
625
  case Op_DivF: case Op_DivD:
626
  case Op_AndI: case Op_AndL:
627
  case Op_OrI:  case Op_OrL:
628
  case Op_XorI: case Op_XorL:
629
  case Op_MulAddS2I:
630
    *start = 1;
631
    *end   = 3; // 2 vector operands
632
    break;
633
  case Op_FmaD:
634
  case Op_FmaF:
635
    *start = 1;
636
    *end   = 4; // 3 vector operands
637
    break;
638
  default:
639
    *start = 1;
640
    *end   = n->req(); // default is all operands
641
  }
642
}
643

644
VectorNode* VectorNode::make_mask_node(int vopc, Node* n1, Node* n2, uint vlen, BasicType bt) {
645
  guarantee(vopc > 0, "vopc must be > 0");
646
  const TypeVect* vmask_type = TypeVect::makemask(bt, vlen);
647
  switch (vopc) {
648
    case Op_AndV:
649
      if (Matcher::match_rule_supported_vector_masked(Op_AndVMask, vlen, bt)) {
650
        return new AndVMaskNode(n1, n2, vmask_type);
651
      }
652
      return new AndVNode(n1, n2, vmask_type);
653
    case Op_OrV:
654
      if (Matcher::match_rule_supported_vector_masked(Op_OrVMask, vlen, bt)) {
655
        return new OrVMaskNode(n1, n2, vmask_type);
656
      }
657
      return new OrVNode(n1, n2, vmask_type);
658
    case Op_XorV:
659
      if (Matcher::match_rule_supported_vector_masked(Op_XorVMask, vlen, bt)) {
660
        return new XorVMaskNode(n1, n2, vmask_type);
661
      }
662
      return new XorVNode(n1, n2, vmask_type);
663
    default:
664
      fatal("Unsupported mask vector creation for '%s'", NodeClassNames[vopc]);
665
      return nullptr;
666
  }
667
}
668

669
// Make a vector node for binary operation
670
VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt, bool is_mask, bool is_var_shift) {
671
  // This method should not be called for unimplemented vectors.
672
  guarantee(vopc > 0, "vopc must be > 0");
673

674
  if (is_mask) {
675
    return make_mask_node(vopc, n1, n2, vt->length(), vt->element_basic_type());
676
  }
677

678
  switch (vopc) {
679
  case Op_AddVB: return new AddVBNode(n1, n2, vt);
680
  case Op_AddVS: return new AddVSNode(n1, n2, vt);
681
  case Op_AddVI: return new AddVINode(n1, n2, vt);
682
  case Op_AddVL: return new AddVLNode(n1, n2, vt);
683
  case Op_AddVF: return new AddVFNode(n1, n2, vt);
684
  case Op_AddVD: return new AddVDNode(n1, n2, vt);
685

686
  case Op_SubVB: return new SubVBNode(n1, n2, vt);
687
  case Op_SubVS: return new SubVSNode(n1, n2, vt);
688
  case Op_SubVI: return new SubVINode(n1, n2, vt);
689
  case Op_SubVL: return new SubVLNode(n1, n2, vt);
690
  case Op_SubVF: return new SubVFNode(n1, n2, vt);
691
  case Op_SubVD: return new SubVDNode(n1, n2, vt);
692

693
  case Op_MulVB: return new MulVBNode(n1, n2, vt);
694
  case Op_MulVS: return new MulVSNode(n1, n2, vt);
695
  case Op_MulVI: return new MulVINode(n1, n2, vt);
696
  case Op_MulVL: return new MulVLNode(n1, n2, vt);
697
  case Op_MulVF: return new MulVFNode(n1, n2, vt);
698
  case Op_MulVD: return new MulVDNode(n1, n2, vt);
699

700
  case Op_DivVF: return new DivVFNode(n1, n2, vt);
701
  case Op_DivVD: return new DivVDNode(n1, n2, vt);
702

703
  case Op_MinV: return new MinVNode(n1, n2, vt);
704
  case Op_MaxV: return new MaxVNode(n1, n2, vt);
705

706
  case Op_AbsVF: return new AbsVFNode(n1, vt);
707
  case Op_AbsVD: return new AbsVDNode(n1, vt);
708
  case Op_AbsVB: return new AbsVBNode(n1, vt);
709
  case Op_AbsVS: return new AbsVSNode(n1, vt);
710
  case Op_AbsVI: return new AbsVINode(n1, vt);
711
  case Op_AbsVL: return new AbsVLNode(n1, vt);
712

713
  case Op_NegVI: return new NegVINode(n1, vt);
714
  case Op_NegVL: return new NegVLNode(n1, vt);
715
  case Op_NegVF: return new NegVFNode(n1, vt);
716
  case Op_NegVD: return new NegVDNode(n1, vt);
717

718
  case Op_ReverseV: return new ReverseVNode(n1, vt);
719
  case Op_ReverseBytesV: return new ReverseBytesVNode(n1, vt);
720

721
  case Op_SqrtVF: return new SqrtVFNode(n1, vt);
722
  case Op_SqrtVD: return new SqrtVDNode(n1, vt);
723

724
  case Op_RoundVF: return new RoundVFNode(n1, vt);
725
  case Op_RoundVD: return new RoundVDNode(n1, vt);
726

727
  case Op_PopCountVI: return new PopCountVINode(n1, vt);
728
  case Op_PopCountVL: return new PopCountVLNode(n1, vt);
729
  case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt);
730
  case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt);
731

732
  case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt, is_var_shift);
733
  case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt, is_var_shift);
734
  case Op_LShiftVI: return new LShiftVINode(n1, n2, vt, is_var_shift);
735
  case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt, is_var_shift);
736

737
  case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt, is_var_shift);
738
  case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt, is_var_shift);
739
  case Op_RShiftVI: return new RShiftVINode(n1, n2, vt, is_var_shift);
740
  case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt, is_var_shift);
741

742
  case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt, is_var_shift);
743
  case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt, is_var_shift);
744
  case Op_URShiftVI: return new URShiftVINode(n1, n2, vt, is_var_shift);
745
  case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt, is_var_shift);
746

747
  case Op_AndV: return new AndVNode(n1, n2, vt);
748
  case Op_OrV:  return new OrVNode (n1, n2, vt);
749
  case Op_XorV: return new XorVNode(n1, n2, vt);
750

751
  case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt);
752

753
  case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt);
754

755
  case Op_ExpandV: return new ExpandVNode(n1, n2, vt);
756
  case Op_CompressV: return new CompressVNode(n1, n2, vt);
757
  case Op_CompressM: assert(n1 == nullptr, ""); return new CompressMNode(n2, vt);
758
  case Op_CompressBitsV: return new CompressBitsVNode(n1, n2, vt);
759
  case Op_ExpandBitsV: return new ExpandBitsVNode(n1, n2, vt);
760
  case Op_CountLeadingZerosV: return new CountLeadingZerosVNode(n1, vt);
761
  case Op_CountTrailingZerosV: return new CountTrailingZerosVNode(n1, vt);
762
  default:
763
    fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
764
    return nullptr;
765
  }
766
}
767

768
// Return the vector version of a scalar binary operation node.
769
VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt, bool is_var_shift) {
770
  const TypeVect* vt = TypeVect::make(bt, vlen);
771
  int vopc = VectorNode::opcode(opc, bt);
772
  // This method should not be called for unimplemented vectors.
773
  guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
774
  return make(vopc, n1, n2, vt, false, is_var_shift);
775
}
776

777
// Make a vector node for ternary operation
778
VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt) {
779
  // This method should not be called for unimplemented vectors.
780
  guarantee(vopc > 0, "vopc must be > 0");
781
  switch (vopc) {
782
  case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
783
  case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
784
  case Op_SignumVD: return new SignumVDNode(n1, n2, n3, vt);
785
  case Op_SignumVF: return new SignumVFNode(n1, n2, n3, vt);
786
  default:
787
    fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
788
    return nullptr;
789
  }
790
}
791

792
// Return the vector version of a scalar ternary operation node.
793
VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
794
  const TypeVect* vt = TypeVect::make(bt, vlen);
795
  int vopc = VectorNode::opcode(opc, bt);
796
  // This method should not be called for unimplemented vectors.
797
  guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
798
  return make(vopc, n1, n2, n3, vt);
799
}
800

801
// Scalar promotion
802
VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t, bool is_mask) {
803
  BasicType bt = opd_t->array_element_basic_type();
804
  if (is_mask && Matcher::match_rule_supported_vector(Op_MaskAll, vlen, bt)) {
805
    const TypeVect* vt = TypeVect::make(opd_t, vlen, true);
806
    return new MaskAllNode(s, vt);
807
  }
808

809
  const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen)
810
                                          : TypeVect::make(bt, vlen);
811
  return new ReplicateNode(s, vt);
812
}
813

814
VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) {
815
  // Match shift count type with shift vector type.
816
  const TypeVect* vt = TypeVect::make(bt, vlen);
817
  switch (opc) {
818
  case Op_LShiftI:
819
  case Op_LShiftL:
820
    return new LShiftCntVNode(cnt, vt);
821
  case Op_RShiftI:
822
  case Op_RShiftL:
823
  case Op_URShiftB:
824
  case Op_URShiftS:
825
  case Op_URShiftI:
826
  case Op_URShiftL:
827
    return new RShiftCntVNode(cnt, vt);
828
  default:
829
    fatal("Missed vector creation for '%s'", NodeClassNames[opc]);
830
    return nullptr;
831
  }
832
}
833

834
bool VectorNode::is_vector_rotate(int opc) {
835
  switch (opc) {
836
  case Op_RotateLeftV:
837
  case Op_RotateRightV:
838
    return true;
839
  default:
840
    return false;
841
  }
842
}
843

844
bool VectorNode::is_vector_integral_negate(int opc) {
845
  return opc == Op_NegVI || opc == Op_NegVL;
846
}
847

848
bool VectorNode::is_vector_shift(int opc) {
849
  assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
850
  switch (opc) {
851
  case Op_LShiftVB:
852
  case Op_LShiftVS:
853
  case Op_LShiftVI:
854
  case Op_LShiftVL:
855
  case Op_RShiftVB:
856
  case Op_RShiftVS:
857
  case Op_RShiftVI:
858
  case Op_RShiftVL:
859
  case Op_URShiftVB:
860
  case Op_URShiftVS:
861
  case Op_URShiftVI:
862
  case Op_URShiftVL:
863
    return true;
864
  default:
865
    return false;
866
  }
867
}
868

869
bool VectorNode::is_vector_shift_count(int opc) {
870
  assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
871
  switch (opc) {
872
  case Op_RShiftCntV:
873
  case Op_LShiftCntV:
874
    return true;
875
  default:
876
    return false;
877
  }
878
}
879

880
static bool is_con(Node* n, long con) {
881
  if (n->is_Con()) {
882
    const Type* t = n->bottom_type();
883
    if (t->isa_int() && t->is_int()->get_con() == (int)con) {
884
      return true;
885
    }
886
    if (t->isa_long() && t->is_long()->get_con() == con) {
887
      return true;
888
    }
889
  }
890
  return false;
891
}
892

893
// Return true if every bit in this vector is 1.
894
bool VectorNode::is_all_ones_vector(Node* n) {
895
  switch (n->Opcode()) {
896
  case Op_Replicate:
897
    return is_integral_type(n->bottom_type()->is_vect()->element_basic_type()) &&
898
           is_con(n->in(1), -1);
899
  case Op_MaskAll:
900
    return is_con(n->in(1), -1);
901
  default:
902
    return false;
903
  }
904
}
905

906
// Return true if every bit in this vector is 0.
907
bool VectorNode::is_all_zeros_vector(Node* n) {
908
  switch (n->Opcode()) {
909
  case Op_Replicate:
910
    return is_integral_type(n->bottom_type()->is_vect()->element_basic_type()) &&
911
           is_con(n->in(1), 0);
912
  case Op_MaskAll:
913
    return is_con(n->in(1), 0);
914
  default:
915
    return false;
916
  }
917
}
918

919
bool VectorNode::is_vector_bitwise_not_pattern(Node* n) {
920
  if (n->Opcode() == Op_XorV) {
921
    return is_all_ones_vector(n->in(1)) ||
922
           is_all_ones_vector(n->in(2));
923
  }
924
  return false;
925
}
926

927
bool VectorNode::is_scalar_unary_op_with_equal_input_and_output_types(int opc) {
928
  switch (opc) {
929
    case Op_SqrtF:
930
    case Op_SqrtD:
931
    case Op_AbsF:
932
    case Op_AbsD:
933
    case Op_AbsI:
934
    case Op_AbsL:
935
    case Op_NegF:
936
    case Op_NegD:
937
    case Op_RoundF:
938
    case Op_RoundD:
939
    case Op_ReverseBytesI:
940
    case Op_ReverseBytesL:
941
    case Op_ReverseBytesUS:
942
    case Op_ReverseBytesS:
943
    case Op_ReverseI:
944
    case Op_ReverseL:
945
    case Op_PopCountI:
946
    case Op_CountLeadingZerosI:
947
    case Op_CountTrailingZerosI:
948
      return true;
949
    default:
950
      return false;
951
  }
952
}
953

954
// Java API for Long.bitCount/numberOfLeadingZeros/numberOfTrailingZeros
955
// returns int type, but Vector API for them returns long type. To unify
956
// the implementation in backend, AutoVectorization splits the vector
957
// implementation for Java API into an execution node with long type plus
958
// another node converting long to int.
959
bool VectorNode::is_scalar_op_that_returns_int_but_vector_op_returns_long(int opc) {
960
  switch (opc) {
961
    case Op_PopCountL:
962
    case Op_CountLeadingZerosL:
963
    case Op_CountTrailingZerosL:
964
      return true;
965
    default:
966
      return false;
967
  }
968
}
969

970

971
Node* VectorNode::try_to_gen_masked_vector(PhaseGVN* gvn, Node* node, const TypeVect* vt) {
972
  int vopc = node->Opcode();
973
  uint vlen = vt->length();
974
  BasicType bt = vt->element_basic_type();
975

976
  // Predicated vectors do not need to add another mask input
977
  if (node->is_predicated_vector() || !Matcher::has_predicated_vectors() ||
978
      !Matcher::match_rule_supported_vector_masked(vopc, vlen, bt) ||
979
      !Matcher::match_rule_supported_vector(Op_VectorMaskGen, vlen, bt)) {
980
    return nullptr;
981
  }
982

983
  Node* mask = nullptr;
984
  // Generate a vector mask for vector operation whose vector length is lower than the
985
  // hardware supported max vector length.
986
  if (vt->length_in_bytes() < (uint)MaxVectorSize) {
987
    Node* length = gvn->transform(new ConvI2LNode(gvn->makecon(TypeInt::make(vlen))));
988
    mask = gvn->transform(VectorMaskGenNode::make(length, bt, vlen));
989
  } else {
990
    return nullptr;
991
  }
992

993
  // Generate the related masked op for vector load/store/load_gather/store_scatter.
994
  // Or append the mask to the vector op's input list by default.
995
  switch(vopc) {
996
  case Op_LoadVector:
997
    return new LoadVectorMaskedNode(node->in(0), node->in(1), node->in(2),
998
                                    node->as_LoadVector()->adr_type(), vt, mask,
999
                                    node->as_LoadVector()->control_dependency());
1000
  case Op_LoadVectorGather:
1001
    return new LoadVectorGatherMaskedNode(node->in(0), node->in(1), node->in(2),
1002
                                          node->as_LoadVector()->adr_type(), vt,
1003
                                          node->in(3), mask);
1004
  case Op_StoreVector:
1005
    return new StoreVectorMaskedNode(node->in(0), node->in(1), node->in(2), node->in(3),
1006
                                     node->as_StoreVector()->adr_type(), mask);
1007
  case Op_StoreVectorScatter:
1008
    return new StoreVectorScatterMaskedNode(node->in(0), node->in(1), node->in(2),
1009
                                            node->as_StoreVector()->adr_type(),
1010
                                            node->in(3), node->in(4), mask);
1011
  default:
1012
    // Add the mask as an additional input to the original vector node by default.
1013
    // This is used for almost all the vector nodes.
1014
    node->add_req(mask);
1015
    node->add_flag(Node::Flag_is_predicated_vector);
1016
    return node;
1017
  }
1018
}
1019

1020
Node* VectorNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1021
  if (Matcher::vector_needs_partial_operations(this, vect_type())) {
1022
    return try_to_gen_masked_vector(phase, this, vect_type());
1023
  }
1024
  return nullptr;
1025
}
1026

1027
// Return initial Pack node. Additional operands added with add_opd() calls.
1028
PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
1029
  const TypeVect* vt = TypeVect::make(bt, vlen);
1030
  switch (bt) {
1031
  case T_BOOLEAN:
1032
  case T_BYTE:
1033
    return new PackBNode(s, vt);
1034
  case T_CHAR:
1035
  case T_SHORT:
1036
    return new PackSNode(s, vt);
1037
  case T_INT:
1038
    return new PackINode(s, vt);
1039
  case T_LONG:
1040
    return new PackLNode(s, vt);
1041
  case T_FLOAT:
1042
    return new PackFNode(s, vt);
1043
  case T_DOUBLE:
1044
    return new PackDNode(s, vt);
1045
  default:
1046
    fatal("Type '%s' is not supported for vectors", type2name(bt));
1047
    return nullptr;
1048
  }
1049
}
1050

1051
// Create a binary tree form for Packs. [lo, hi) (half-open) range
1052
PackNode* PackNode::binary_tree_pack(int lo, int hi) {
1053
  int ct = hi - lo;
1054
  assert(is_power_of_2(ct), "power of 2");
1055
  if (ct == 2) {
1056
    PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type());
1057
    pk->add_opd(in(lo+1));
1058
    return pk;
1059
  } else {
1060
    int mid = lo + ct/2;
1061
    PackNode* n1 = binary_tree_pack(lo,  mid);
1062
    PackNode* n2 = binary_tree_pack(mid, hi );
1063

1064
    BasicType bt = n1->vect_type()->element_basic_type();
1065
    assert(bt == n2->vect_type()->element_basic_type(), "should be the same");
1066
    switch (bt) {
1067
    case T_BOOLEAN:
1068
    case T_BYTE:
1069
      return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2));
1070
    case T_CHAR:
1071
    case T_SHORT:
1072
      return new PackINode(n1, n2, TypeVect::make(T_INT, 2));
1073
    case T_INT:
1074
      return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2));
1075
    case T_LONG:
1076
      return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2));
1077
    case T_FLOAT:
1078
      return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
1079
    case T_DOUBLE:
1080
      return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
1081
    default:
1082
      fatal("Type '%s' is not supported for vectors", type2name(bt));
1083
      return nullptr;
1084
    }
1085
  }
1086
}
1087

1088
// Return the vector version of a scalar load node.
1089
LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem,
1090
                                     Node* adr, const TypePtr* atyp,
1091
                                     uint vlen, BasicType bt,
1092
                                     ControlDependency control_dependency) {
1093
  const TypeVect* vt = TypeVect::make(bt, vlen);
1094
  return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency);
1095
}
1096

1097
Node* LoadVectorNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1098
  const TypeVect* vt = vect_type();
1099
  if (Matcher::vector_needs_partial_operations(this, vt)) {
1100
    return VectorNode::try_to_gen_masked_vector(phase, this, vt);
1101
  }
1102
  return LoadNode::Ideal(phase, can_reshape);
1103
}
1104

1105
// Return the vector version of a scalar store node.
1106
StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem, Node* adr,
1107
                                       const TypePtr* atyp, Node* val, uint vlen) {
1108
  return new StoreVectorNode(ctl, mem, adr, atyp, val);
1109
}
1110

1111
Node* StoreVectorNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1112
  const TypeVect* vt = vect_type();
1113
  if (Matcher::vector_needs_partial_operations(this, vt)) {
1114
    return VectorNode::try_to_gen_masked_vector(phase, this, vt);
1115
  }
1116
  return StoreNode::Ideal(phase, can_reshape);
1117
}
1118

1119
Node* LoadVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1120
  if (!in(3)->is_top() && in(3)->Opcode() == Op_VectorMaskGen) {
1121
    Node* mask_len = in(3)->in(1);
1122
    const TypeLong* ty = phase->type(mask_len)->isa_long();
1123
    if (ty && ty->is_con()) {
1124
      BasicType mask_bt = Matcher::vector_element_basic_type(in(3));
1125
      int load_sz = type2aelembytes(mask_bt) * ty->get_con();
1126
      assert(load_sz <= MaxVectorSize, "Unexpected load size");
1127
      if (load_sz == MaxVectorSize) {
1128
        Node* ctr = in(MemNode::Control);
1129
        Node* mem = in(MemNode::Memory);
1130
        Node* adr = in(MemNode::Address);
1131
        return phase->transform(new LoadVectorNode(ctr, mem, adr, adr_type(), vect_type()));
1132
      }
1133
    }
1134
  }
1135
  return LoadVectorNode::Ideal(phase, can_reshape);
1136
}
1137

1138
Node* StoreVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1139
  if (!in(4)->is_top() && in(4)->Opcode() == Op_VectorMaskGen) {
1140
    Node* mask_len = in(4)->in(1);
1141
    const TypeLong* ty = phase->type(mask_len)->isa_long();
1142
    if (ty && ty->is_con()) {
1143
      BasicType mask_bt = Matcher::vector_element_basic_type(in(4));
1144
      int load_sz = type2aelembytes(mask_bt) * ty->get_con();
1145
      assert(load_sz <= MaxVectorSize, "Unexpected store size");
1146
      if (load_sz == MaxVectorSize) {
1147
        Node* ctr = in(MemNode::Control);
1148
        Node* mem = in(MemNode::Memory);
1149
        Node* adr = in(MemNode::Address);
1150
        Node* val = in(MemNode::ValueIn);
1151
        return phase->transform(new StoreVectorNode(ctr, mem, adr, adr_type(), val));
1152
      }
1153
    }
1154
  }
1155
  return StoreVectorNode::Ideal(phase, can_reshape);
1156
}
1157

1158
int ExtractNode::opcode(BasicType bt) {
1159
  switch (bt) {
1160
    case T_BOOLEAN: return Op_ExtractUB;
1161
    case T_BYTE:    return Op_ExtractB;
1162
    case T_CHAR:    return Op_ExtractC;
1163
    case T_SHORT:   return Op_ExtractS;
1164
    case T_INT:     return Op_ExtractI;
1165
    case T_LONG:    return Op_ExtractL;
1166
    case T_FLOAT:   return Op_ExtractF;
1167
    case T_DOUBLE:  return Op_ExtractD;
1168
    default:
1169
      assert(false, "wrong type: %s", type2name(bt));
1170
      return 0;
1171
  }
1172
}
1173

1174
// Extract a scalar element of vector by constant position.
1175
Node* ExtractNode::make(Node* v, ConINode* pos, BasicType bt) {
1176
  assert(pos->get_int() >= 0 &&
1177
         pos->get_int() < Matcher::max_vector_size(bt), "pos in range");
1178
  switch (bt) {
1179
  case T_BOOLEAN: return new ExtractUBNode(v, pos);
1180
  case T_BYTE:    return new ExtractBNode(v, pos);
1181
  case T_CHAR:    return new ExtractCNode(v, pos);
1182
  case T_SHORT:   return new ExtractSNode(v, pos);
1183
  case T_INT:     return new ExtractINode(v, pos);
1184
  case T_LONG:    return new ExtractLNode(v, pos);
1185
  case T_FLOAT:   return new ExtractFNode(v, pos);
1186
  case T_DOUBLE:  return new ExtractDNode(v, pos);
1187
  default:
1188
    assert(false, "wrong type: %s", type2name(bt));
1189
    return nullptr;
1190
  }
1191
}
1192

1193
int ReductionNode::opcode(int opc, BasicType bt) {
1194
  int vopc = opc;
1195
  switch (opc) {
1196
    case Op_AddI:
1197
      switch (bt) {
1198
        case T_BOOLEAN:
1199
        case T_CHAR: return 0;
1200
        case T_BYTE:
1201
        case T_SHORT:
1202
        case T_INT:
1203
          vopc = Op_AddReductionVI;
1204
          break;
1205
        default: ShouldNotReachHere(); return 0;
1206
      }
1207
      break;
1208
    case Op_AddL:
1209
      assert(bt == T_LONG, "must be");
1210
      vopc = Op_AddReductionVL;
1211
      break;
1212
    case Op_AddF:
1213
      assert(bt == T_FLOAT, "must be");
1214
      vopc = Op_AddReductionVF;
1215
      break;
1216
    case Op_AddD:
1217
      assert(bt == T_DOUBLE, "must be");
1218
      vopc = Op_AddReductionVD;
1219
      break;
1220
    case Op_MulI:
1221
      switch (bt) {
1222
        case T_BOOLEAN:
1223
        case T_CHAR: return 0;
1224
        case T_BYTE:
1225
        case T_SHORT:
1226
        case T_INT:
1227
          vopc = Op_MulReductionVI;
1228
          break;
1229
        default: ShouldNotReachHere(); return 0;
1230
      }
1231
      break;
1232
    case Op_MulL:
1233
      assert(bt == T_LONG, "must be");
1234
      vopc = Op_MulReductionVL;
1235
      break;
1236
    case Op_MulF:
1237
      assert(bt == T_FLOAT, "must be");
1238
      vopc = Op_MulReductionVF;
1239
      break;
1240
    case Op_MulD:
1241
      assert(bt == T_DOUBLE, "must be");
1242
      vopc = Op_MulReductionVD;
1243
      break;
1244
    case Op_MinI:
1245
      switch (bt) {
1246
        case T_BOOLEAN:
1247
        case T_CHAR: return 0;
1248
        case T_BYTE:
1249
        case T_SHORT:
1250
        case T_INT:
1251
          vopc = Op_MinReductionV;
1252
          break;
1253
        default: ShouldNotReachHere(); return 0;
1254
      }
1255
      break;
1256
    case Op_MinL:
1257
      assert(bt == T_LONG, "must be");
1258
      vopc = Op_MinReductionV;
1259
      break;
1260
    case Op_MinF:
1261
      assert(bt == T_FLOAT, "must be");
1262
      vopc = Op_MinReductionV;
1263
      break;
1264
    case Op_MinD:
1265
      assert(bt == T_DOUBLE, "must be");
1266
      vopc = Op_MinReductionV;
1267
      break;
1268
    case Op_MaxI:
1269
      switch (bt) {
1270
        case T_BOOLEAN:
1271
        case T_CHAR: return 0;
1272
        case T_BYTE:
1273
        case T_SHORT:
1274
        case T_INT:
1275
          vopc = Op_MaxReductionV;
1276
          break;
1277
        default: ShouldNotReachHere(); return 0;
1278
      }
1279
      break;
1280
    case Op_MaxL:
1281
      assert(bt == T_LONG, "must be");
1282
      vopc = Op_MaxReductionV;
1283
      break;
1284
    case Op_MaxF:
1285
      assert(bt == T_FLOAT, "must be");
1286
      vopc = Op_MaxReductionV;
1287
      break;
1288
    case Op_MaxD:
1289
      assert(bt == T_DOUBLE, "must be");
1290
      vopc = Op_MaxReductionV;
1291
      break;
1292
    case Op_AndI:
1293
      switch (bt) {
1294
      case T_BOOLEAN:
1295
      case T_CHAR: return 0;
1296
      case T_BYTE:
1297
      case T_SHORT:
1298
      case T_INT:
1299
        vopc = Op_AndReductionV;
1300
        break;
1301
      default: ShouldNotReachHere(); return 0;
1302
      }
1303
      break;
1304
    case Op_AndL:
1305
      assert(bt == T_LONG, "must be");
1306
      vopc = Op_AndReductionV;
1307
      break;
1308
    case Op_OrI:
1309
      switch(bt) {
1310
      case T_BOOLEAN:
1311
      case T_CHAR: return 0;
1312
      case T_BYTE:
1313
      case T_SHORT:
1314
      case T_INT:
1315
        vopc = Op_OrReductionV;
1316
        break;
1317
      default: ShouldNotReachHere(); return 0;
1318
      }
1319
      break;
1320
    case Op_OrL:
1321
      assert(bt == T_LONG, "must be");
1322
      vopc = Op_OrReductionV;
1323
      break;
1324
    case Op_XorI:
1325
      switch(bt) {
1326
      case T_BOOLEAN:
1327
      case T_CHAR: return 0;
1328
      case T_BYTE:
1329
      case T_SHORT:
1330
      case T_INT:
1331
        vopc = Op_XorReductionV;
1332
        break;
1333
      default: ShouldNotReachHere(); return 0;
1334
      }
1335
      break;
1336
    case Op_XorL:
1337
      assert(bt == T_LONG, "must be");
1338
      vopc = Op_XorReductionV;
1339
      break;
1340
    default:
1341
      break;
1342
  }
1343
  return vopc;
1344
}
1345

1346
// Return the appropriate reduction node.
1347
ReductionNode* ReductionNode::make(int opc, Node* ctrl, Node* n1, Node* n2, BasicType bt,
1348
                                   bool requires_strict_order) {
1349

1350
  int vopc = opcode(opc, bt);
1351

1352
  // This method should not be called for unimplemented vectors.
1353
  guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]);
1354

1355
  switch (vopc) {
1356
  case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2);
1357
  case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2);
1358
  case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2, requires_strict_order);
1359
  case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2, requires_strict_order);
1360
  case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2);
1361
  case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2);
1362
  case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2, requires_strict_order);
1363
  case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2, requires_strict_order);
1364
  case Op_MinReductionV:  return new MinReductionVNode (ctrl, n1, n2);
1365
  case Op_MaxReductionV:  return new MaxReductionVNode (ctrl, n1, n2);
1366
  case Op_AndReductionV:  return new AndReductionVNode (ctrl, n1, n2);
1367
  case Op_OrReductionV:   return new OrReductionVNode  (ctrl, n1, n2);
1368
  case Op_XorReductionV:  return new XorReductionVNode (ctrl, n1, n2);
1369
  default:
1370
    assert(false, "unknown node: %s", NodeClassNames[vopc]);
1371
    return nullptr;
1372
  }
1373
}
1374

1375
Node* ReductionNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1376
  const TypeVect* vt = vect_type();
1377
  if (Matcher::vector_needs_partial_operations(this, vt)) {
1378
    return VectorNode::try_to_gen_masked_vector(phase, this, vt);
1379
  }
1380
  return nullptr;
1381
}
1382

1383
Node* VectorLoadMaskNode::Identity(PhaseGVN* phase) {
1384
  BasicType out_bt = type()->is_vect()->element_basic_type();
1385
  if (!Matcher::has_predicated_vectors() && out_bt == T_BOOLEAN) {
1386
    return in(1); // redundant conversion
1387
  }
1388

1389
  return this;
1390
}
1391

1392
Node* VectorStoreMaskNode::Identity(PhaseGVN* phase) {
1393
  // Identity transformation on boolean vectors.
1394
  //   VectorStoreMask (VectorLoadMask bv) elem_size ==> bv
1395
  //   vector[n]{bool} => vector[n]{t} => vector[n]{bool}
1396
  if (in(1)->Opcode() == Op_VectorLoadMask) {
1397
    return in(1)->in(1);
1398
  }
1399
  return this;
1400
}
1401

1402
VectorStoreMaskNode* VectorStoreMaskNode::make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem) {
1403
  assert(in->bottom_type()->isa_vect(), "sanity");
1404
  const TypeVect* vt = TypeVect::make(T_BOOLEAN, num_elem);
1405
  int elem_size = type2aelembytes(in_type);
1406
  return new VectorStoreMaskNode(in, gvn.intcon(elem_size), vt);
1407
}
1408

1409
VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) {
1410
  const TypeVect* vt = TypeVect::make(bt, vlen);
1411
  switch (vopc) {
1412
    case Op_VectorCastB2X:  return new VectorCastB2XNode(n1, vt);
1413
    case Op_VectorCastS2X:  return new VectorCastS2XNode(n1, vt);
1414
    case Op_VectorCastI2X:  return new VectorCastI2XNode(n1, vt);
1415
    case Op_VectorCastL2X:  return new VectorCastL2XNode(n1, vt);
1416
    case Op_VectorCastF2X:  return new VectorCastF2XNode(n1, vt);
1417
    case Op_VectorCastD2X:  return new VectorCastD2XNode(n1, vt);
1418
    case Op_VectorUCastB2X: return new VectorUCastB2XNode(n1, vt);
1419
    case Op_VectorUCastS2X: return new VectorUCastS2XNode(n1, vt);
1420
    case Op_VectorUCastI2X: return new VectorUCastI2XNode(n1, vt);
1421
    case Op_VectorCastHF2F: return new VectorCastHF2FNode(n1, vt);
1422
    case Op_VectorCastF2HF: return new VectorCastF2HFNode(n1, vt);
1423
    default:
1424
      assert(false, "unknown node: %s", NodeClassNames[vopc]);
1425
      return nullptr;
1426
  }
1427
}
1428

1429
int VectorCastNode::opcode(int sopc, BasicType bt, bool is_signed) {
1430
  assert((is_integral_type(bt) && bt != T_LONG) || is_signed, "");
1431

1432
  // Handle special case for to/from Half Float conversions
1433
  switch (sopc) {
1434
    case Op_ConvHF2F:
1435
      assert(bt == T_SHORT, "");
1436
      return Op_VectorCastHF2F;
1437
    case Op_ConvF2HF:
1438
      assert(bt == T_FLOAT, "");
1439
      return Op_VectorCastF2HF;
1440
    default:
1441
      // Handled normally below
1442
      break;
1443
  }
1444

1445
  // Handle normal conversions
1446
  switch (bt) {
1447
    case T_BYTE:   return is_signed ? Op_VectorCastB2X : Op_VectorUCastB2X;
1448
    case T_SHORT:  return is_signed ? Op_VectorCastS2X : Op_VectorUCastS2X;
1449
    case T_INT:    return is_signed ? Op_VectorCastI2X : Op_VectorUCastI2X;
1450
    case T_LONG:   return Op_VectorCastL2X;
1451
    case T_FLOAT:  return Op_VectorCastF2X;
1452
    case T_DOUBLE: return Op_VectorCastD2X;
1453
    default:
1454
      assert(bt == T_CHAR || bt == T_BOOLEAN, "unknown type: %s", type2name(bt));
1455
      return 0;
1456
  }
1457
}
1458

1459
bool VectorCastNode::implemented(int opc, uint vlen, BasicType src_type, BasicType dst_type) {
1460
  if (is_java_primitive(dst_type) &&
1461
      is_java_primitive(src_type) &&
1462
      (vlen > 1) && is_power_of_2(vlen) &&
1463
      VectorNode::vector_size_supported_auto_vectorization(dst_type, vlen)) {
1464
    int vopc = VectorCastNode::opcode(opc, src_type);
1465
    return vopc > 0 && Matcher::match_rule_supported_auto_vectorization(vopc, vlen, dst_type);
1466
  }
1467
  return false;
1468
}
1469

1470
Node* VectorCastNode::Identity(PhaseGVN* phase) {
1471
  if (!in(1)->is_top()) {
1472
    BasicType  in_bt = in(1)->bottom_type()->is_vect()->element_basic_type();
1473
    BasicType out_bt = vect_type()->element_basic_type();
1474
    if (in_bt == out_bt) {
1475
      return in(1); // redundant cast
1476
    }
1477
  }
1478
  return this;
1479
}
1480

1481
Node* ReductionNode::make_identity_con_scalar(PhaseGVN& gvn, int sopc, BasicType bt) {
1482
  int vopc = opcode(sopc, bt);
1483
  guarantee(vopc != sopc, "Vector reduction for '%s' is not implemented", NodeClassNames[sopc]);
1484

1485
  switch (vopc) {
1486
    case Op_AndReductionV:
1487
      switch (bt) {
1488
        case T_BYTE:
1489
        case T_SHORT:
1490
        case T_INT:
1491
          return gvn.makecon(TypeInt::MINUS_1);
1492
        case T_LONG:
1493
          return gvn.makecon(TypeLong::MINUS_1);
1494
        default:
1495
          fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]);
1496
          return nullptr;
1497
      }
1498
      break;
1499
    case Op_AddReductionVI: // fallthrough
1500
    case Op_AddReductionVL: // fallthrough
1501
    case Op_AddReductionVF: // fallthrough
1502
    case Op_AddReductionVD:
1503
    case Op_OrReductionV:
1504
    case Op_XorReductionV:
1505
      return gvn.zerocon(bt);
1506
    case Op_MulReductionVI:
1507
      return gvn.makecon(TypeInt::ONE);
1508
    case Op_MulReductionVL:
1509
      return gvn.makecon(TypeLong::ONE);
1510
    case Op_MulReductionVF:
1511
      return gvn.makecon(TypeF::ONE);
1512
    case Op_MulReductionVD:
1513
      return gvn.makecon(TypeD::ONE);
1514
    case Op_MinReductionV:
1515
      switch (bt) {
1516
        case T_BYTE:
1517
          return gvn.makecon(TypeInt::make(max_jbyte));
1518
        case T_SHORT:
1519
          return gvn.makecon(TypeInt::make(max_jshort));
1520
        case T_INT:
1521
          return gvn.makecon(TypeInt::MAX);
1522
        case T_LONG:
1523
          return gvn.makecon(TypeLong::MAX);
1524
        case T_FLOAT:
1525
          return gvn.makecon(TypeF::POS_INF);
1526
        case T_DOUBLE:
1527
          return gvn.makecon(TypeD::POS_INF);
1528
          default: Unimplemented(); return nullptr;
1529
      }
1530
      break;
1531
    case Op_MaxReductionV:
1532
      switch (bt) {
1533
        case T_BYTE:
1534
          return gvn.makecon(TypeInt::make(min_jbyte));
1535
        case T_SHORT:
1536
          return gvn.makecon(TypeInt::make(min_jshort));
1537
        case T_INT:
1538
          return gvn.makecon(TypeInt::MIN);
1539
        case T_LONG:
1540
          return gvn.makecon(TypeLong::MIN);
1541
        case T_FLOAT:
1542
          return gvn.makecon(TypeF::NEG_INF);
1543
        case T_DOUBLE:
1544
          return gvn.makecon(TypeD::NEG_INF);
1545
          default: Unimplemented(); return nullptr;
1546
      }
1547
      break;
1548
    default:
1549
      fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
1550
      return nullptr;
1551
  }
1552
}
1553

1554
bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) {
1555
  if (is_java_primitive(bt) &&
1556
      (vlen > 1) && is_power_of_2(vlen) &&
1557
      VectorNode::vector_size_supported_auto_vectorization(bt, vlen)) {
1558
    int vopc = ReductionNode::opcode(opc, bt);
1559
    return vopc != opc && Matcher::match_rule_supported_auto_vectorization(vopc, vlen, bt);
1560
  }
1561
  return false;
1562
}
1563

1564
MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3,
1565
                                       Node* mask, uint truth_table, const TypeVect* vt) {
1566
  assert(truth_table <= 0xFF, "invalid");
1567
  assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1568
  assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1569
  assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1570
  assert(!mask || mask->bottom_type()->isa_vectmask(), "predicated register type expected");
1571
  Node* fn = gvn.intcon(truth_table);
1572
  return new MacroLogicVNode(in1, in2, in3, fn, mask, vt);
1573
}
1574

1575
Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
1576
                                           int vlen, BasicType bt, PhaseGVN* phase) {
1577
  assert(is_integral_type(bt), "sanity");
1578
  const TypeVect* vt = TypeVect::make(bt, vlen);
1579

1580
  int shift_mask = (type2aelembytes(bt) * 8) - 1;
1581
  int shiftLOpc = (bt == T_LONG) ? Op_LShiftL : Op_LShiftI;
1582
  auto urshiftopc = [=]() {
1583
    switch(bt) {
1584
      case T_INT: return Op_URShiftI;
1585
      case T_LONG: return Op_URShiftL;
1586
      case T_BYTE: return Op_URShiftB;
1587
      case T_SHORT: return Op_URShiftS;
1588
      default: return (Opcodes)0;
1589
    }
1590
  };
1591
  int shiftROpc = urshiftopc();
1592

1593
  // Compute shift values for right rotation and
1594
  // later swap them in case of left rotation.
1595
  Node* shiftRCnt = nullptr;
1596
  Node* shiftLCnt = nullptr;
1597
  const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
1598
  bool is_binary_vector_op = false;
1599
  if (cnt_type && cnt_type->is_con()) {
1600
    // Constant shift.
1601
    int shift = cnt_type->get_con() & shift_mask;
1602
    shiftRCnt = phase->intcon(shift);
1603
    shiftLCnt = phase->intcon(shift_mask + 1 - shift);
1604
  } else if (cnt->Opcode() == Op_Replicate) {
1605
    // Scalar variable shift, handle replicates generated by auto vectorizer.
1606
    cnt = cnt->in(1);
1607
    if (bt == T_LONG) {
1608
      // Shift count vector for Rotate vector has long elements too.
1609
      if (cnt->Opcode() == Op_ConvI2L) {
1610
         cnt = cnt->in(1);
1611
      } else {
1612
         assert(cnt->bottom_type()->isa_long() &&
1613
                cnt->bottom_type()->is_long()->is_con(), "Long constant expected");
1614
         cnt = phase->transform(new ConvL2INode(cnt));
1615
      }
1616
    }
1617
    shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(shift_mask)));
1618
    shiftLCnt = phase->transform(new SubINode(phase->intcon(shift_mask + 1), shiftRCnt));
1619
  } else {
1620
    // Variable vector rotate count.
1621
    assert(Matcher::supports_vector_variable_shifts(), "");
1622

1623
    int subVopc = 0;
1624
    int addVopc = 0;
1625
    Node* shift_mask_node = nullptr;
1626
    Node* const_one_node = nullptr;
1627

1628
    assert(cnt->bottom_type()->isa_vect(), "Unexpected shift");
1629
    const Type* elem_ty = Type::get_const_basic_type(bt);
1630

1631
    if (bt == T_LONG) {
1632
      shift_mask_node = phase->longcon(shift_mask);
1633
      const_one_node = phase->longcon(1L);
1634
      subVopc = VectorNode::opcode(Op_SubL, bt);
1635
      addVopc = VectorNode::opcode(Op_AddL, bt);
1636
    } else {
1637
      shift_mask_node = phase->intcon(shift_mask);
1638
      const_one_node = phase->intcon(1);
1639
      subVopc = VectorNode::opcode(Op_SubI, bt);
1640
      addVopc = VectorNode::opcode(Op_AddI, bt);
1641
    }
1642
    Node* vector_mask = phase->transform(VectorNode::scalar2vector(shift_mask_node, vlen, elem_ty));
1643
    Node* vector_one = phase->transform(VectorNode::scalar2vector(const_one_node, vlen, elem_ty));
1644

1645
    shiftRCnt = cnt;
1646
    shiftRCnt = phase->transform(VectorNode::make(Op_AndV, shiftRCnt, vector_mask, vt));
1647
    vector_mask = phase->transform(VectorNode::make(addVopc, vector_one, vector_mask, vt));
1648
    shiftLCnt = phase->transform(VectorNode::make(subVopc, vector_mask, shiftRCnt, vt));
1649
    is_binary_vector_op = true;
1650
  }
1651

1652
  // Swap the computed left and right shift counts.
1653
  if (is_rotate_left) {
1654
    swap(shiftRCnt,shiftLCnt);
1655
  }
1656

1657
  if (!is_binary_vector_op) {
1658
    shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
1659
    shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
1660
  }
1661

1662
  return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt, is_binary_vector_op)),
1663
                     phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt, is_binary_vector_op)),
1664
                     vt);
1665
}
1666

1667
Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1668
  int vlen = length();
1669
  BasicType bt = vect_type()->element_basic_type();
1670
  if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1671
       !Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) {
1672
    return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase);
1673
  }
1674
  return nullptr;
1675
}
1676

1677
Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1678
  int vlen = length();
1679
  BasicType bt = vect_type()->element_basic_type();
1680
  if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1681
       !Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) {
1682
    return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase);
1683
  }
1684
  return nullptr;
1685
}
1686

1687
#ifndef PRODUCT
1688
void VectorMaskCmpNode::dump_spec(outputStream *st) const {
1689
  st->print(" %d #", _predicate); _type->dump_on(st);
1690
}
1691
#endif // PRODUCT
1692

1693
Node* VectorReinterpretNode::Identity(PhaseGVN *phase) {
1694
  Node* n = in(1);
1695
  if (n->Opcode() == Op_VectorReinterpret) {
1696
    // "VectorReinterpret (VectorReinterpret node) ==> node" if:
1697
    //   1) Types of 'node' and 'this' are identical
1698
    //   2) Truncations are not introduced by the first VectorReinterpret
1699
    if (Type::equals(bottom_type(), n->in(1)->bottom_type()) &&
1700
        length_in_bytes() <= n->bottom_type()->is_vect()->length_in_bytes()) {
1701
      return n->in(1);
1702
    }
1703
  }
1704
  return this;
1705
}
1706

1707
Node* VectorInsertNode::make(Node* vec, Node* new_val, int position, PhaseGVN& gvn) {
1708
  assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range");
1709
  ConINode* pos = gvn.intcon(position);
1710
  return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect());
1711
}
1712

1713
Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1714
  Node* n = obj()->uncast();
1715
  if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1716
    if (Type::equals(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type())) {
1717
      // Handled by VectorUnboxNode::Identity()
1718
    } else {
1719
      VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
1720
      ciKlass* vbox_klass = vbox->box_type()->instance_klass();
1721
      const TypeVect* in_vt = vbox->vec_type();
1722
      const TypeVect* out_vt = type()->is_vect();
1723

1724
      if (in_vt->length() == out_vt->length()) {
1725
        Node* value = vbox->in(VectorBoxNode::Value);
1726

1727
        bool is_vector_mask    = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
1728
        bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
1729
        if (is_vector_mask) {
1730
          // VectorUnbox (VectorBox vmask) ==> VectorMaskCast vmask
1731
          const TypeVect* vmask_type = TypeVect::makemask(out_vt->element_basic_type(), out_vt->length());
1732
          return new VectorMaskCastNode(value, vmask_type);
1733
        } else if (is_vector_shuffle) {
1734
          if (!is_shuffle_to_vector()) {
1735
            // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle
1736
            return new VectorLoadShuffleNode(value, out_vt);
1737
          }
1738
        } else {
1739
          // Vector type mismatch is only supported for masks and shuffles, but sometimes it happens in pathological cases.
1740
        }
1741
      } else {
1742
        // Vector length mismatch.
1743
        // Sometimes happen in pathological cases (e.g., when unboxing happens in effectively dead code).
1744
      }
1745
    }
1746
  }
1747
  return nullptr;
1748
}
1749

1750
Node* VectorUnboxNode::Identity(PhaseGVN* phase) {
1751
  Node* n = obj()->uncast();
1752
  if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1753
    if (Type::equals(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type())) {
1754
      return n->in(VectorBoxNode::Value); // VectorUnbox (VectorBox v) ==> v
1755
    } else {
1756
      // Handled by VectorUnboxNode::Ideal().
1757
    }
1758
  }
1759
  return this;
1760
}
1761

1762
const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) {
1763
  const Type** fields = TypeTuple::fields(0);
1764
  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
1765

1766
  fields = TypeTuple::fields(1);
1767
  fields[TypeFunc::Parms+0] = box_type;
1768
  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
1769

1770
  return TypeFunc::make(domain, range);
1771
}
1772

1773
Node* ShiftVNode::Identity(PhaseGVN* phase) {
1774
  Node* in2 = in(2);
1775
  // Shift by ZERO does nothing
1776
  if (is_vshift_cnt(in2) && phase->find_int_type(in2->in(1)) == TypeInt::ZERO) {
1777
    return in(1);
1778
  }
1779
  return this;
1780
}
1781

1782
Node* VectorMaskGenNode::make(Node* length, BasicType mask_bt) {
1783
  int max_vector = Matcher::max_vector_size(mask_bt);
1784
  return make(length, mask_bt, max_vector);
1785
}
1786

1787
Node* VectorMaskGenNode::make(Node* length, BasicType mask_bt, int mask_len) {
1788
  const TypeVectMask* t_vmask = TypeVectMask::make(mask_bt, mask_len);
1789
  return new VectorMaskGenNode(length, t_vmask);
1790
}
1791

1792
Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) {
1793
  switch(mopc) {
1794
    case Op_VectorMaskTrueCount:
1795
      return new VectorMaskTrueCountNode(mask, ty);
1796
    case Op_VectorMaskLastTrue:
1797
      return new VectorMaskLastTrueNode(mask, ty);
1798
    case Op_VectorMaskFirstTrue:
1799
      return new VectorMaskFirstTrueNode(mask, ty);
1800
    case Op_VectorMaskToLong:
1801
      return new VectorMaskToLongNode(mask, ty);
1802
    default:
1803
      assert(false, "Unhandled operation");
1804
  }
1805
  return nullptr;
1806
}
1807

1808
Node* VectorMaskOpNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1809
  const TypeVect* vt = vect_type();
1810
  if (Matcher::vector_needs_partial_operations(this, vt)) {
1811
    return VectorNode::try_to_gen_masked_vector(phase, this, vt);
1812
  }
1813
  return nullptr;
1814
}
1815

1816
Node* VectorMaskToLongNode::Identity(PhaseGVN* phase) {
1817
  if (in(1)->Opcode() == Op_VectorLongToMask) {
1818
    return in(1)->in(1);
1819
  }
1820
  return this;
1821
}
1822

1823
Node* VectorLongToMaskNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1824
  const TypeVect* dst_type = bottom_type()->is_vect();
1825
  if (in(1)->Opcode() == Op_AndL &&
1826
      in(1)->in(1)->Opcode() == Op_VectorMaskToLong &&
1827
      in(1)->in(2)->bottom_type()->isa_long() &&
1828
      in(1)->in(2)->bottom_type()->is_long()->is_con() &&
1829
      in(1)->in(2)->bottom_type()->is_long()->get_con() == ((1L << dst_type->length()) - 1)) {
1830
      // Different src/dst mask length represents a re-interpretation operation,
1831
      // we can however generate a mask casting operation if length matches.
1832
     Node* src = in(1)->in(1)->in(1);
1833
     if (dst_type->isa_vectmask() == nullptr) {
1834
       if (src->Opcode() != Op_VectorStoreMask) {
1835
         return nullptr;
1836
       }
1837
       src = src->in(1);
1838
     }
1839
     const TypeVect* src_type = src->bottom_type()->is_vect();
1840
     if (src_type->length() == dst_type->length() &&
1841
         ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask() == nullptr) ||
1842
          (src_type->isa_vectmask() && dst_type->isa_vectmask()))) {
1843
       return new VectorMaskCastNode(src, dst_type);
1844
     }
1845
  }
1846
  return nullptr;
1847
}
1848

1849
Node* FmaVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1850
  // We canonicalize the node by converting "(-a)*b+c" into "b*(-a)+c"
1851
  // This reduces the number of rules in the matcher, as we only need to check
1852
  // for negations on the second argument, and not the symmetric case where
1853
  // the first argument is negated.
1854
  // We cannot do this if the FmaV is masked, since the inactive lanes have to return
1855
  // the first input (i.e. "-a"). If we were to swap the inputs, the inactive lanes would
1856
  // incorrectly return "b".
1857
  if (!is_predicated_vector() && in(1)->is_NegV() && !in(2)->is_NegV()) {
1858
    swap_edges(1, 2);
1859
    return this;
1860
  }
1861
  return nullptr;
1862
}
1863

1864
// Generate other vector nodes to implement the masked/non-masked vector negation.
1865
Node* NegVNode::degenerate_integral_negate(PhaseGVN* phase, bool is_predicated) {
1866
  const TypeVect* vt = vect_type();
1867
  BasicType bt = vt->element_basic_type();
1868
  uint vlen = length();
1869

1870
  // Transformation for predicated NegVI/L
1871
  if (is_predicated) {
1872
      // (NegVI/L src m) ==> (AddVI/L (XorV src (ReplicateI/L -1) m) (ReplicateI/L 1) m)
1873
      Node* const_minus_one = nullptr;
1874
      Node* const_one = nullptr;
1875
      int add_opc;
1876
      if (bt == T_LONG) {
1877
        const_minus_one = phase->longcon(-1L);
1878
        const_one = phase->longcon(1L);
1879
        add_opc = Op_AddL;
1880
      } else {
1881
        const_minus_one = phase->intcon(-1);
1882
        const_one = phase->intcon(1);
1883
        add_opc = Op_AddI;
1884
      }
1885
      const_minus_one = phase->transform(VectorNode::scalar2vector(const_minus_one, vlen, Type::get_const_basic_type(bt)));
1886
      Node* xorv = VectorNode::make(Op_XorV, in(1), const_minus_one, vt);
1887
      xorv->add_req(in(2));
1888
      xorv->add_flag(Node::Flag_is_predicated_vector);
1889
      phase->transform(xorv);
1890
      const_one = phase->transform(VectorNode::scalar2vector(const_one, vlen, Type::get_const_basic_type(bt)));
1891
      Node* addv = VectorNode::make(VectorNode::opcode(add_opc, bt), xorv, const_one, vt);
1892
      addv->add_req(in(2));
1893
      addv->add_flag(Node::Flag_is_predicated_vector);
1894
      return addv;
1895
  }
1896

1897
  // NegVI/L ==> (SubVI/L (ReplicateI/L 0) src)
1898
  Node* const_zero = nullptr;
1899
  int sub_opc;
1900
  if (bt == T_LONG) {
1901
    const_zero = phase->longcon(0L);
1902
    sub_opc = Op_SubL;
1903
  } else {
1904
    const_zero = phase->intcon(0);
1905
    sub_opc = Op_SubI;
1906
  }
1907
  const_zero = phase->transform(VectorNode::scalar2vector(const_zero, vlen, Type::get_const_basic_type(bt)));
1908
  return VectorNode::make(VectorNode::opcode(sub_opc, bt), const_zero, in(1), vt);
1909
}
1910

1911
Node* NegVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1912
  BasicType bt = vect_type()->element_basic_type();
1913
  uint vlen = length();
1914
  int opc = Opcode();
1915
  if (is_vector_integral_negate(opc)) {
1916
    if (is_predicated_vector()) {
1917
      if (!Matcher::match_rule_supported_vector_masked(opc, vlen, bt)) {
1918
        return degenerate_integral_negate(phase, true);
1919
      }
1920
    } else if (!Matcher::match_rule_supported_vector(opc, vlen, bt)) {
1921
      return degenerate_integral_negate(phase, false);
1922
    }
1923
  }
1924
  return nullptr;
1925
}
1926

1927
static Node* reverse_operations_identity(Node* n, Node* in1) {
1928
  if (n->is_predicated_using_blend()) {
1929
    return n;
1930
  }
1931
  if (n->Opcode() == in1->Opcode()) {
1932
    // OperationV (OperationV X MASK) MASK =>  X
1933
    if (n->is_predicated_vector() && in1->is_predicated_vector() && n->in(2) == in1->in(2)) {
1934
      return in1->in(1);
1935
    // OperationV (OperationV X) =>  X
1936
    } else if (!n->is_predicated_vector() && !in1->is_predicated_vector())  {
1937
      return in1->in(1);
1938
    }
1939
  }
1940
  return n;
1941
}
1942

1943
Node* ReverseBytesVNode::Identity(PhaseGVN* phase) {
1944
  // "(ReverseBytesV X) => X" if the element type is T_BYTE.
1945
  if (vect_type()->element_basic_type() == T_BYTE) {
1946
    return in(1);
1947
  }
1948
  return reverse_operations_identity(this, in(1));
1949
}
1950

1951
Node* ReverseVNode::Identity(PhaseGVN* phase) {
1952
  return reverse_operations_identity(this, in(1));
1953
}
1954

1955
// Optimize away redundant AndV/OrV nodes when the operation
1956
// is applied on the same input node multiple times
1957
static Node* redundant_logical_identity(Node* n) {
1958
  Node* n1 = n->in(1);
1959
  // (OperationV (OperationV src1 src2) src1) => (OperationV src1 src2)
1960
  // (OperationV (OperationV src1 src2) src2) => (OperationV src1 src2)
1961
  // (OperationV (OperationV src1 src2 m1) src1 m1) => (OperationV src1 src2 m1)
1962
  // (OperationV (OperationV src1 src2 m1) src2 m1) => (OperationV src1 src2 m1)
1963
  if (n->Opcode() == n1->Opcode()) {
1964
    if (((!n->is_predicated_vector() && !n1->is_predicated_vector()) ||
1965
         ( n->is_predicated_vector() &&  n1->is_predicated_vector() && n->in(3) == n1->in(3))) &&
1966
         ( n->in(2) == n1->in(1) || n->in(2) == n1->in(2))) {
1967
      return n1;
1968
    }
1969
  }
1970

1971
  Node* n2 = n->in(2);
1972
  if (n->Opcode() == n2->Opcode()) {
1973
    // (OperationV src1 (OperationV src1 src2)) => OperationV(src1, src2)
1974
    // (OperationV src2 (OperationV src1 src2)) => OperationV(src1, src2)
1975
    // (OperationV src1 (OperationV src1 src2 m1) m1) => OperationV(src1 src2 m1)
1976
    // It is not possible to optimize - (OperationV src2 (OperationV src1 src2 m1) m1) as the
1977
    // results of both "OperationV" nodes are different for unmasked lanes
1978
    if ((!n->is_predicated_vector() && !n2->is_predicated_vector() &&
1979
         (n->in(1) == n2->in(1) || n->in(1) == n2->in(2))) ||
1980
         (n->is_predicated_vector() && n2->is_predicated_vector() && n->in(3) == n2->in(3) &&
1981
         n->in(1) == n2->in(1))) {
1982
      return n2;
1983
    }
1984
  }
1985

1986
  return n;
1987
}
1988

1989
Node* AndVNode::Identity(PhaseGVN* phase) {
1990
  // (AndV src (Replicate m1))   => src
1991
  // (AndVMask src (MaskAll m1)) => src
1992
  if (VectorNode::is_all_ones_vector(in(2))) {
1993
    return in(1);
1994
  }
1995
  // (AndV (Replicate zero) src)   => (Replicate zero)
1996
  // (AndVMask (MaskAll zero) src) => (MaskAll zero)
1997
  if (VectorNode::is_all_zeros_vector(in(1))) {
1998
    return in(1);
1999
  }
2000
  // The following transformations are only applied to
2001
  // the un-predicated operation, since the VectorAPI
2002
  // masked operation requires the unmasked lanes to
2003
  // save the same values in the first operand.
2004
  if (!is_predicated_vector()) {
2005
    // (AndV (Replicate m1) src)   => src
2006
    // (AndVMask (MaskAll m1) src) => src
2007
    if (VectorNode::is_all_ones_vector(in(1))) {
2008
      return in(2);
2009
    }
2010
    // (AndV src (Replicate zero))   => (Replicate zero)
2011
    // (AndVMask src (MaskAll zero)) => (MaskAll zero)
2012
    if (VectorNode::is_all_zeros_vector(in(2))) {
2013
      return in(2);
2014
    }
2015
  }
2016

2017
  // (AndV src src)     => src
2018
  // (AndVMask src src) => src
2019
  if (in(1) == in(2)) {
2020
    return in(1);
2021
  }
2022
  return redundant_logical_identity(this);
2023
}
2024

2025
Node* OrVNode::Identity(PhaseGVN* phase) {
2026
  // (OrV (Replicate m1) src)   => (Replicate m1)
2027
  // (OrVMask (MaskAll m1) src) => (MaskAll m1)
2028
  if (VectorNode::is_all_ones_vector(in(1))) {
2029
    return in(1);
2030
  }
2031
  // (OrV src (Replicate zero))   => src
2032
  // (OrVMask src (MaskAll zero)) => src
2033
  if (VectorNode::is_all_zeros_vector(in(2))) {
2034
    return in(1);
2035
  }
2036
  // The following transformations are only applied to
2037
  // the un-predicated operation, since the VectorAPI
2038
  // masked operation requires the unmasked lanes to
2039
  // save the same values in the first operand.
2040
  if (!is_predicated_vector()) {
2041
    // (OrV src (Replicate m1))   => (Replicate m1)
2042
    // (OrVMask src (MaskAll m1)) => (MaskAll m1)
2043
    if (VectorNode::is_all_ones_vector(in(2))) {
2044
      return in(2);
2045
    }
2046
    // (OrV (Replicate zero) src)   => src
2047
    // (OrVMask (MaskAll zero) src) => src
2048
    if (VectorNode::is_all_zeros_vector(in(1))) {
2049
      return in(2);
2050
    }
2051
  }
2052

2053
  // (OrV src src)     => src
2054
  // (OrVMask src src) => src
2055
  if (in(1) == in(2)) {
2056
    return in(1);
2057
  }
2058
  return redundant_logical_identity(this);
2059
}
2060

2061
Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2062
  // (XorV src src)      => (Replicate zero)
2063
  // (XorVMask src src)  => (MaskAll zero)
2064
  //
2065
  // The transformation is only applied to the un-predicated
2066
  // operation, since the VectorAPI masked operation requires
2067
  // the unmasked lanes to save the same values in the first
2068
  // operand.
2069
  if (!is_predicated_vector() && (in(1) == in(2))) {
2070
    BasicType bt = vect_type()->element_basic_type();
2071
    Node* zero = phase->transform(phase->zerocon(bt));
2072
    return VectorNode::scalar2vector(zero, length(), Type::get_const_basic_type(bt),
2073
                                     bottom_type()->isa_vectmask() != nullptr);
2074
  }
2075
  return nullptr;
2076
}
2077

2078
Node* VectorBlendNode::Identity(PhaseGVN* phase) {
2079
  // (VectorBlend X X MASK) => X
2080
  if (in(1) == in(2)) {
2081
    return in(1);
2082
  }
2083
  return this;
2084
}
2085

2086
#ifndef PRODUCT
2087
void VectorBoxAllocateNode::dump_spec(outputStream *st) const {
2088
  CallStaticJavaNode::dump_spec(st);
2089
}
2090

2091
#endif // !PRODUCT
2092