jdk

1
/*
2
 * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4
 *
5
 * This code is free software; you can redistribute it and/or modify it
6
 * under the terms of the GNU General Public License version 2 only, as
7
 * published by the Free Software Foundation.
8
 *
9
 * This code is distributed in the hope that it will be useful, but WITHOUT
10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12
 * version 2 for more details (a copy is included in the LICENSE file that
13
 * accompanied this code).
14
 *
15
 * You should have received a copy of the GNU General Public License version
16
 * 2 along with this work; if not, write to the Free Software Foundation,
17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18
 *
19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20
 * or visit www.oracle.com if you need additional information or have any
21
 * questions.
22
 *
23
 */
24

25
#include "precompiled.hpp"
26
#include "asm/macroAssembler.hpp"
27
#include "asm/macroAssembler.inline.hpp"
28
#include "c1/c1_CodeStubs.hpp"
29
#include "c1/c1_Compilation.hpp"
30
#include "c1/c1_LIRAssembler.hpp"
31
#include "c1/c1_MacroAssembler.hpp"
32
#include "c1/c1_Runtime1.hpp"
33
#include "c1/c1_ValueStack.hpp"
34
#include "ci/ciArrayKlass.hpp"
35
#include "ci/ciInstance.hpp"
36
#include "compiler/oopMap.hpp"
37
#include "gc/shared/collectedHeap.hpp"
38
#include "gc/shared/gc_globals.hpp"
39
#include "nativeInst_x86.hpp"
40
#include "oops/objArrayKlass.hpp"
41
#include "runtime/frame.inline.hpp"
42
#include "runtime/safepointMechanism.hpp"
43
#include "runtime/sharedRuntime.hpp"
44
#include "runtime/stubRoutines.hpp"
45
#include "utilities/powerOfTwo.hpp"
46
#include "vmreg_x86.inline.hpp"
47

48

49
// These masks are used to provide 128-bit aligned bitmasks to the XMM
50
// instructions, to allow sign-masking or sign-bit flipping.  They allow
51
// fast versions of NegF/NegD and AbsF/AbsD.
52

53
// Note: 'double' and 'long long' have 32-bits alignment on x86.
54
static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
55
  // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
56
  // of 128-bits operands for SSE instructions.
57
  jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
58
  // Store the value to a 128-bits operand.
59
  operand[0] = lo;
60
  operand[1] = hi;
61
  return operand;
62
}
63

64
// Buffer for 128-bits masks used by SSE instructions.
65
static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
66

67
// Static initialization during VM startup.
68
static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2],         CONST64(0x7FFFFFFF7FFFFFFF),         CONST64(0x7FFFFFFF7FFFFFFF));
69
static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],         CONST64(0x7FFFFFFFFFFFFFFF),         CONST64(0x7FFFFFFFFFFFFFFF));
70
static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
71
static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
72

73

74
NEEDS_CLEANUP // remove this definitions ?
75
const Register SYNC_header = rax;   // synchronization header
76
const Register SHIFT_count = rcx;   // where count for shift operations must be
77

78
#define __ _masm->
79

80

81
static void select_different_registers(Register preserve,
82
                                       Register extra,
83
                                       Register &tmp1,
84
                                       Register &tmp2) {
85
  if (tmp1 == preserve) {
86
    assert_different_registers(tmp1, tmp2, extra);
87
    tmp1 = extra;
88
  } else if (tmp2 == preserve) {
89
    assert_different_registers(tmp1, tmp2, extra);
90
    tmp2 = extra;
91
  }
92
  assert_different_registers(preserve, tmp1, tmp2);
93
}
94

95

96

97
static void select_different_registers(Register preserve,
98
                                       Register extra,
99
                                       Register &tmp1,
100
                                       Register &tmp2,
101
                                       Register &tmp3) {
102
  if (tmp1 == preserve) {
103
    assert_different_registers(tmp1, tmp2, tmp3, extra);
104
    tmp1 = extra;
105
  } else if (tmp2 == preserve) {
106
    assert_different_registers(tmp1, tmp2, tmp3, extra);
107
    tmp2 = extra;
108
  } else if (tmp3 == preserve) {
109
    assert_different_registers(tmp1, tmp2, tmp3, extra);
110
    tmp3 = extra;
111
  }
112
  assert_different_registers(preserve, tmp1, tmp2, tmp3);
113
}
114

115

116

117
bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
118
  if (opr->is_constant()) {
119
    LIR_Const* constant = opr->as_constant_ptr();
120
    switch (constant->type()) {
121
      case T_INT: {
122
        return true;
123
      }
124

125
      default:
126
        return false;
127
    }
128
  }
129
  return false;
130
}
131

132

133
LIR_Opr LIR_Assembler::receiverOpr() {
134
  return FrameMap::receiver_opr;
135
}
136

137
LIR_Opr LIR_Assembler::osrBufferPointer() {
138
  return FrameMap::as_pointer_opr(receiverOpr()->as_register());
139
}
140

141
//--------------fpu register translations-----------------------
142

143

144
address LIR_Assembler::float_constant(float f) {
145
  address const_addr = __ float_constant(f);
146
  if (const_addr == nullptr) {
147
    bailout("const section overflow");
148
    return __ code()->consts()->start();
149
  } else {
150
    return const_addr;
151
  }
152
}
153

154

155
address LIR_Assembler::double_constant(double d) {
156
  address const_addr = __ double_constant(d);
157
  if (const_addr == nullptr) {
158
    bailout("const section overflow");
159
    return __ code()->consts()->start();
160
  } else {
161
    return const_addr;
162
  }
163
}
164

165
#ifndef _LP64
166
void LIR_Assembler::fpop() {
167
  __ fpop();
168
}
169

170
void LIR_Assembler::fxch(int i) {
171
  __ fxch(i);
172
}
173

174
void LIR_Assembler::fld(int i) {
175
  __ fld_s(i);
176
}
177

178
void LIR_Assembler::ffree(int i) {
179
  __ ffree(i);
180
}
181
#endif // !_LP64
182

183
void LIR_Assembler::breakpoint() {
184
  __ int3();
185
}
186

187
void LIR_Assembler::push(LIR_Opr opr) {
188
  if (opr->is_single_cpu()) {
189
    __ push_reg(opr->as_register());
190
  } else if (opr->is_double_cpu()) {
191
    NOT_LP64(__ push_reg(opr->as_register_hi()));
192
    __ push_reg(opr->as_register_lo());
193
  } else if (opr->is_stack()) {
194
    __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
195
  } else if (opr->is_constant()) {
196
    LIR_Const* const_opr = opr->as_constant_ptr();
197
    if (const_opr->type() == T_OBJECT) {
198
      __ push_oop(const_opr->as_jobject(), rscratch1);
199
    } else if (const_opr->type() == T_INT) {
200
      __ push_jint(const_opr->as_jint());
201
    } else {
202
      ShouldNotReachHere();
203
    }
204

205
  } else {
206
    ShouldNotReachHere();
207
  }
208
}
209

210
void LIR_Assembler::pop(LIR_Opr opr) {
211
  if (opr->is_single_cpu()) {
212
    __ pop_reg(opr->as_register());
213
  } else {
214
    ShouldNotReachHere();
215
  }
216
}
217

218
bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
219
  return addr->base()->is_illegal() && addr->index()->is_illegal();
220
}
221

222
//-------------------------------------------
223

224
Address LIR_Assembler::as_Address(LIR_Address* addr) {
225
  return as_Address(addr, rscratch1);
226
}
227

228
Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
229
  if (addr->base()->is_illegal()) {
230
    assert(addr->index()->is_illegal(), "must be illegal too");
231
    AddressLiteral laddr((address)addr->disp(), relocInfo::none);
232
    if (! __ reachable(laddr)) {
233
      __ movptr(tmp, laddr.addr());
234
      Address res(tmp, 0);
235
      return res;
236
    } else {
237
      return __ as_Address(laddr);
238
    }
239
  }
240

241
  Register base = addr->base()->as_pointer_register();
242

243
  if (addr->index()->is_illegal()) {
244
    return Address( base, addr->disp());
245
  } else if (addr->index()->is_cpu_register()) {
246
    Register index = addr->index()->as_pointer_register();
247
    return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
248
  } else if (addr->index()->is_constant()) {
249
    intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
250
    assert(Assembler::is_simm32(addr_offset), "must be");
251

252
    return Address(base, addr_offset);
253
  } else {
254
    Unimplemented();
255
    return Address();
256
  }
257
}
258

259

260
Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
261
  Address base = as_Address(addr);
262
  return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
263
}
264

265

266
Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
267
  return as_Address(addr);
268
}
269

270

271
void LIR_Assembler::osr_entry() {
272
  offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
273
  BlockBegin* osr_entry = compilation()->hir()->osr_entry();
274
  ValueStack* entry_state = osr_entry->state();
275
  int number_of_locks = entry_state->locks_size();
276

277
  // we jump here if osr happens with the interpreter
278
  // state set up to continue at the beginning of the
279
  // loop that triggered osr - in particular, we have
280
  // the following registers setup:
281
  //
282
  // rcx: osr buffer
283
  //
284

285
  // build frame
286
  ciMethod* m = compilation()->method();
287
  __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
288

289
  // OSR buffer is
290
  //
291
  // locals[nlocals-1..0]
292
  // monitors[0..number_of_locks]
293
  //
294
  // locals is a direct copy of the interpreter frame so in the osr buffer
295
  // so first slot in the local array is the last local from the interpreter
296
  // and last slot is local[0] (receiver) from the interpreter
297
  //
298
  // Similarly with locks. The first lock slot in the osr buffer is the nth lock
299
  // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
300
  // in the interpreter frame (the method lock if a sync method)
301

302
  // Initialize monitors in the compiled activation.
303
  //   rcx: pointer to osr buffer
304
  //
305
  // All other registers are dead at this point and the locals will be
306
  // copied into place by code emitted in the IR.
307

308
  Register OSR_buf = osrBufferPointer()->as_pointer_register();
309
  { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
310
    int monitor_offset = BytesPerWord * method()->max_locals() +
311
      (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
312
    // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
313
    // the OSR buffer using 2 word entries: first the lock and then
314
    // the oop.
315
    for (int i = 0; i < number_of_locks; i++) {
316
      int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
317
#ifdef ASSERT
318
      // verify the interpreter's monitor has a non-null object
319
      {
320
        Label L;
321
        __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
322
        __ jcc(Assembler::notZero, L);
323
        __ stop("locked object is null");
324
        __ bind(L);
325
      }
326
#endif
327
      __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
328
      __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
329
      __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
330
      __ movptr(frame_map()->address_for_monitor_object(i), rbx);
331
    }
332
  }
333
}
334

335

336
// inline cache check; done before the frame is built.
337
int LIR_Assembler::check_icache() {
338
  return __ ic_check(CodeEntryAlignment);
339
}
340

341
void LIR_Assembler::clinit_barrier(ciMethod* method) {
342
  assert(VM_Version::supports_fast_class_init_checks(), "sanity");
343
  assert(!method->holder()->is_not_initialized(), "initialization should have been started");
344

345
  Label L_skip_barrier;
346
  Register klass = rscratch1;
347
  Register thread = LP64_ONLY( r15_thread ) NOT_LP64( noreg );
348
  assert(thread != noreg, "x86_32 not implemented");
349

350
  __ mov_metadata(klass, method->holder()->constant_encoding());
351
  __ clinit_barrier(klass, thread, &L_skip_barrier /*L_fast_path*/);
352

353
  __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
354

355
  __ bind(L_skip_barrier);
356
}
357

358
void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
359
  jobject o = nullptr;
360
  PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
361
  __ movoop(reg, o);
362
  patching_epilog(patch, lir_patch_normal, reg, info);
363
}
364

365
void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
366
  Metadata* o = nullptr;
367
  PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
368
  __ mov_metadata(reg, o);
369
  patching_epilog(patch, lir_patch_normal, reg, info);
370
}
371

372
// This specifies the rsp decrement needed to build the frame
373
int LIR_Assembler::initial_frame_size_in_bytes() const {
374
  // if rounding, must let FrameMap know!
375

376
  // The frame_map records size in slots (32bit word)
377

378
  // subtract two words to account for return address and link
379
  return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
380
}
381

382

383
int LIR_Assembler::emit_exception_handler() {
384
  // generate code for exception handler
385
  address handler_base = __ start_a_stub(exception_handler_size());
386
  if (handler_base == nullptr) {
387
    // not enough space left for the handler
388
    bailout("exception handler overflow");
389
    return -1;
390
  }
391

392
  int offset = code_offset();
393

394
  // the exception oop and pc are in rax, and rdx
395
  // no other registers need to be preserved, so invalidate them
396
  __ invalidate_registers(false, true, true, false, true, true);
397

398
  // check that there is really an exception
399
  __ verify_not_null_oop(rax);
400

401
  // search an exception handler (rax: exception oop, rdx: throwing pc)
402
  __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
403
  __ should_not_reach_here();
404
  guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
405
  __ end_a_stub();
406

407
  return offset;
408
}
409

410

411
// Emit the code to remove the frame from the stack in the exception
412
// unwind path.
413
int LIR_Assembler::emit_unwind_handler() {
414
#ifndef PRODUCT
415
  if (CommentedAssembly) {
416
    _masm->block_comment("Unwind handler");
417
  }
418
#endif
419

420
  int offset = code_offset();
421

422
  // Fetch the exception from TLS and clear out exception related thread state
423
  Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
424
  NOT_LP64(__ get_thread(thread));
425
  __ movptr(rax, Address(thread, JavaThread::exception_oop_offset()));
426
  __ movptr(Address(thread, JavaThread::exception_oop_offset()), NULL_WORD);
427
  __ movptr(Address(thread, JavaThread::exception_pc_offset()), NULL_WORD);
428

429
  __ bind(_unwind_handler_entry);
430
  __ verify_not_null_oop(rax);
431
  if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
432
    __ mov(rbx, rax);  // Preserve the exception (rbx is always callee-saved)
433
  }
434

435
  // Perform needed unlocking
436
  MonitorExitStub* stub = nullptr;
437
  if (method()->is_synchronized()) {
438
    monitor_address(0, FrameMap::rax_opr);
439
    stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
440
    if (LockingMode == LM_MONITOR) {
441
      __ jmp(*stub->entry());
442
    } else {
443
      __ unlock_object(rdi, rsi, rax, *stub->entry());
444
    }
445
    __ bind(*stub->continuation());
446
  }
447

448
  if (compilation()->env()->dtrace_method_probes()) {
449
#ifdef _LP64
450
    __ mov(rdi, r15_thread);
451
    __ mov_metadata(rsi, method()->constant_encoding());
452
#else
453
    __ get_thread(rax);
454
    __ movptr(Address(rsp, 0), rax);
455
    __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
456
#endif
457
    __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
458
  }
459

460
  if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
461
    __ mov(rax, rbx);  // Restore the exception
462
  }
463

464
  // remove the activation and dispatch to the unwind handler
465
  __ remove_frame(initial_frame_size_in_bytes());
466
  __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
467

468
  // Emit the slow path assembly
469
  if (stub != nullptr) {
470
    stub->emit_code(this);
471
  }
472

473
  return offset;
474
}
475

476

477
int LIR_Assembler::emit_deopt_handler() {
478
  // generate code for exception handler
479
  address handler_base = __ start_a_stub(deopt_handler_size());
480
  if (handler_base == nullptr) {
481
    // not enough space left for the handler
482
    bailout("deopt handler overflow");
483
    return -1;
484
  }
485

486
  int offset = code_offset();
487
  InternalAddress here(__ pc());
488

489
  __ pushptr(here.addr(), rscratch1);
490
  __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
491
  guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
492
  __ end_a_stub();
493

494
  return offset;
495
}
496

497
void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
498
  assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
499
  if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
500
    assert(result->fpu() == 0, "result must already be on TOS");
501
  }
502

503
  // Pop the stack before the safepoint code
504
  __ remove_frame(initial_frame_size_in_bytes());
505

506
  if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
507
    __ reserved_stack_check();
508
  }
509

510
  // Note: we do not need to round double result; float result has the right precision
511
  // the poll sets the condition code, but no data registers
512

513
#ifdef _LP64
514
  const Register thread = r15_thread;
515
#else
516
  const Register thread = rbx;
517
  __ get_thread(thread);
518
#endif
519
  code_stub->set_safepoint_offset(__ offset());
520
  __ relocate(relocInfo::poll_return_type);
521
  __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
522
  __ ret(0);
523
}
524

525

526
int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
527
  guarantee(info != nullptr, "Shouldn't be null");
528
  int offset = __ offset();
529
#ifdef _LP64
530
  const Register poll_addr = rscratch1;
531
  __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
532
#else
533
  assert(tmp->is_cpu_register(), "needed");
534
  const Register poll_addr = tmp->as_register();
535
  __ get_thread(poll_addr);
536
  __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
537
#endif
538
  add_debug_info_for_branch(info);
539
  __ relocate(relocInfo::poll_type);
540
  address pre_pc = __ pc();
541
  __ testl(rax, Address(poll_addr, 0));
542
  address post_pc = __ pc();
543
  guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
544
  return offset;
545
}
546

547

548
void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
549
  if (from_reg != to_reg) __ mov(to_reg, from_reg);
550
}
551

552
void LIR_Assembler::swap_reg(Register a, Register b) {
553
  __ xchgptr(a, b);
554
}
555

556

557
void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
558
  assert(src->is_constant(), "should not call otherwise");
559
  assert(dest->is_register(), "should not call otherwise");
560
  LIR_Const* c = src->as_constant_ptr();
561

562
  switch (c->type()) {
563
    case T_INT: {
564
      assert(patch_code == lir_patch_none, "no patching handled here");
565
      __ movl(dest->as_register(), c->as_jint());
566
      break;
567
    }
568

569
    case T_ADDRESS: {
570
      assert(patch_code == lir_patch_none, "no patching handled here");
571
      __ movptr(dest->as_register(), c->as_jint());
572
      break;
573
    }
574

575
    case T_LONG: {
576
      assert(patch_code == lir_patch_none, "no patching handled here");
577
#ifdef _LP64
578
      __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
579
#else
580
      __ movptr(dest->as_register_lo(), c->as_jint_lo());
581
      __ movptr(dest->as_register_hi(), c->as_jint_hi());
582
#endif // _LP64
583
      break;
584
    }
585

586
    case T_OBJECT: {
587
      if (patch_code != lir_patch_none) {
588
        jobject2reg_with_patching(dest->as_register(), info);
589
      } else {
590
        __ movoop(dest->as_register(), c->as_jobject());
591
      }
592
      break;
593
    }
594

595
    case T_METADATA: {
596
      if (patch_code != lir_patch_none) {
597
        klass2reg_with_patching(dest->as_register(), info);
598
      } else {
599
        __ mov_metadata(dest->as_register(), c->as_metadata());
600
      }
601
      break;
602
    }
603

604
    case T_FLOAT: {
605
      if (dest->is_single_xmm()) {
606
        if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_float()) {
607
          __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
608
        } else {
609
          __ movflt(dest->as_xmm_float_reg(),
610
                   InternalAddress(float_constant(c->as_jfloat())));
611
        }
612
      } else {
613
#ifndef _LP64
614
        assert(dest->is_single_fpu(), "must be");
615
        assert(dest->fpu_regnr() == 0, "dest must be TOS");
616
        if (c->is_zero_float()) {
617
          __ fldz();
618
        } else if (c->is_one_float()) {
619
          __ fld1();
620
        } else {
621
          __ fld_s (InternalAddress(float_constant(c->as_jfloat())));
622
        }
623
#else
624
        ShouldNotReachHere();
625
#endif // !_LP64
626
      }
627
      break;
628
    }
629

630
    case T_DOUBLE: {
631
      if (dest->is_double_xmm()) {
632
        if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_double()) {
633
          __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
634
        } else {
635
          __ movdbl(dest->as_xmm_double_reg(),
636
                    InternalAddress(double_constant(c->as_jdouble())));
637
        }
638
      } else {
639
#ifndef _LP64
640
        assert(dest->is_double_fpu(), "must be");
641
        assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
642
        if (c->is_zero_double()) {
643
          __ fldz();
644
        } else if (c->is_one_double()) {
645
          __ fld1();
646
        } else {
647
          __ fld_d (InternalAddress(double_constant(c->as_jdouble())));
648
        }
649
#else
650
        ShouldNotReachHere();
651
#endif // !_LP64
652
      }
653
      break;
654
    }
655

656
    default:
657
      ShouldNotReachHere();
658
  }
659
}
660

661
void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
662
  assert(src->is_constant(), "should not call otherwise");
663
  assert(dest->is_stack(), "should not call otherwise");
664
  LIR_Const* c = src->as_constant_ptr();
665

666
  switch (c->type()) {
667
    case T_INT:  // fall through
668
    case T_FLOAT:
669
      __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
670
      break;
671

672
    case T_ADDRESS:
673
      __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
674
      break;
675

676
    case T_OBJECT:
677
      __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
678
      break;
679

680
    case T_LONG:  // fall through
681
    case T_DOUBLE:
682
#ifdef _LP64
683
      __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
684
                                              lo_word_offset_in_bytes),
685
                (intptr_t)c->as_jlong_bits(),
686
                rscratch1);
687
#else
688
      __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
689
                                              lo_word_offset_in_bytes), c->as_jint_lo_bits());
690
      __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
691
                                              hi_word_offset_in_bytes), c->as_jint_hi_bits());
692
#endif // _LP64
693
      break;
694

695
    default:
696
      ShouldNotReachHere();
697
  }
698
}
699

700
void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
701
  assert(src->is_constant(), "should not call otherwise");
702
  assert(dest->is_address(), "should not call otherwise");
703
  LIR_Const* c = src->as_constant_ptr();
704
  LIR_Address* addr = dest->as_address_ptr();
705

706
  int null_check_here = code_offset();
707
  switch (type) {
708
    case T_INT:    // fall through
709
    case T_FLOAT:
710
      __ movl(as_Address(addr), c->as_jint_bits());
711
      break;
712

713
    case T_ADDRESS:
714
      __ movptr(as_Address(addr), c->as_jint_bits());
715
      break;
716

717
    case T_OBJECT:  // fall through
718
    case T_ARRAY:
719
      if (c->as_jobject() == nullptr) {
720
        if (UseCompressedOops && !wide) {
721
          __ movl(as_Address(addr), NULL_WORD);
722
        } else {
723
#ifdef _LP64
724
          __ xorptr(rscratch1, rscratch1);
725
          null_check_here = code_offset();
726
          __ movptr(as_Address(addr), rscratch1);
727
#else
728
          __ movptr(as_Address(addr), NULL_WORD);
729
#endif
730
        }
731
      } else {
732
        if (is_literal_address(addr)) {
733
          ShouldNotReachHere();
734
          __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
735
        } else {
736
#ifdef _LP64
737
          __ movoop(rscratch1, c->as_jobject());
738
          if (UseCompressedOops && !wide) {
739
            __ encode_heap_oop(rscratch1);
740
            null_check_here = code_offset();
741
            __ movl(as_Address_lo(addr), rscratch1);
742
          } else {
743
            null_check_here = code_offset();
744
            __ movptr(as_Address_lo(addr), rscratch1);
745
          }
746
#else
747
          __ movoop(as_Address(addr), c->as_jobject(), noreg);
748
#endif
749
        }
750
      }
751
      break;
752

753
    case T_LONG:    // fall through
754
    case T_DOUBLE:
755
#ifdef _LP64
756
      if (is_literal_address(addr)) {
757
        ShouldNotReachHere();
758
        __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
759
      } else {
760
        __ movptr(r10, (intptr_t)c->as_jlong_bits());
761
        null_check_here = code_offset();
762
        __ movptr(as_Address_lo(addr), r10);
763
      }
764
#else
765
      // Always reachable in 32bit so this doesn't produce useless move literal
766
      __ movptr(as_Address_hi(addr), c->as_jint_hi_bits());
767
      __ movptr(as_Address_lo(addr), c->as_jint_lo_bits());
768
#endif // _LP64
769
      break;
770

771
    case T_BOOLEAN: // fall through
772
    case T_BYTE:
773
      __ movb(as_Address(addr), c->as_jint() & 0xFF);
774
      break;
775

776
    case T_CHAR:    // fall through
777
    case T_SHORT:
778
      __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
779
      break;
780

781
    default:
782
      ShouldNotReachHere();
783
  };
784

785
  if (info != nullptr) {
786
    add_debug_info_for_null_check(null_check_here, info);
787
  }
788
}
789

790

791
void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
792
  assert(src->is_register(), "should not call otherwise");
793
  assert(dest->is_register(), "should not call otherwise");
794

795
  // move between cpu-registers
796
  if (dest->is_single_cpu()) {
797
#ifdef _LP64
798
    if (src->type() == T_LONG) {
799
      // Can do LONG -> OBJECT
800
      move_regs(src->as_register_lo(), dest->as_register());
801
      return;
802
    }
803
#endif
804
    assert(src->is_single_cpu(), "must match");
805
    if (src->type() == T_OBJECT) {
806
      __ verify_oop(src->as_register());
807
    }
808
    move_regs(src->as_register(), dest->as_register());
809

810
  } else if (dest->is_double_cpu()) {
811
#ifdef _LP64
812
    if (is_reference_type(src->type())) {
813
      // Surprising to me but we can see move of a long to t_object
814
      __ verify_oop(src->as_register());
815
      move_regs(src->as_register(), dest->as_register_lo());
816
      return;
817
    }
818
#endif
819
    assert(src->is_double_cpu(), "must match");
820
    Register f_lo = src->as_register_lo();
821
    Register f_hi = src->as_register_hi();
822
    Register t_lo = dest->as_register_lo();
823
    Register t_hi = dest->as_register_hi();
824
#ifdef _LP64
825
    assert(f_hi == f_lo, "must be same");
826
    assert(t_hi == t_lo, "must be same");
827
    move_regs(f_lo, t_lo);
828
#else
829
    assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation");
830

831

832
    if (f_lo == t_hi && f_hi == t_lo) {
833
      swap_reg(f_lo, f_hi);
834
    } else if (f_hi == t_lo) {
835
      assert(f_lo != t_hi, "overwriting register");
836
      move_regs(f_hi, t_hi);
837
      move_regs(f_lo, t_lo);
838
    } else {
839
      assert(f_hi != t_lo, "overwriting register");
840
      move_regs(f_lo, t_lo);
841
      move_regs(f_hi, t_hi);
842
    }
843
#endif // LP64
844

845
#ifndef _LP64
846
    // special moves from fpu-register to xmm-register
847
    // necessary for method results
848
  } else if (src->is_single_xmm() && !dest->is_single_xmm()) {
849
    __ movflt(Address(rsp, 0), src->as_xmm_float_reg());
850
    __ fld_s(Address(rsp, 0));
851
  } else if (src->is_double_xmm() && !dest->is_double_xmm()) {
852
    __ movdbl(Address(rsp, 0), src->as_xmm_double_reg());
853
    __ fld_d(Address(rsp, 0));
854
  } else if (dest->is_single_xmm() && !src->is_single_xmm()) {
855
    __ fstp_s(Address(rsp, 0));
856
    __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0));
857
  } else if (dest->is_double_xmm() && !src->is_double_xmm()) {
858
    __ fstp_d(Address(rsp, 0));
859
    __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0));
860
#endif // !_LP64
861

862
    // move between xmm-registers
863
  } else if (dest->is_single_xmm()) {
864
    assert(src->is_single_xmm(), "must match");
865
    __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
866
  } else if (dest->is_double_xmm()) {
867
    assert(src->is_double_xmm(), "must match");
868
    __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
869

870
#ifndef _LP64
871
    // move between fpu-registers (no instruction necessary because of fpu-stack)
872
  } else if (dest->is_single_fpu() || dest->is_double_fpu()) {
873
    assert(src->is_single_fpu() || src->is_double_fpu(), "must match");
874
    assert(src->fpu() == dest->fpu(), "currently should be nothing to do");
875
#endif // !_LP64
876

877
  } else {
878
    ShouldNotReachHere();
879
  }
880
}
881

882
void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
883
  assert(src->is_register(), "should not call otherwise");
884
  assert(dest->is_stack(), "should not call otherwise");
885

886
  if (src->is_single_cpu()) {
887
    Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
888
    if (is_reference_type(type)) {
889
      __ verify_oop(src->as_register());
890
      __ movptr (dst, src->as_register());
891
    } else if (type == T_METADATA || type == T_ADDRESS) {
892
      __ movptr (dst, src->as_register());
893
    } else {
894
      __ movl (dst, src->as_register());
895
    }
896

897
  } else if (src->is_double_cpu()) {
898
    Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
899
    Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
900
    __ movptr (dstLO, src->as_register_lo());
901
    NOT_LP64(__ movptr (dstHI, src->as_register_hi()));
902

903
  } else if (src->is_single_xmm()) {
904
    Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
905
    __ movflt(dst_addr, src->as_xmm_float_reg());
906

907
  } else if (src->is_double_xmm()) {
908
    Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
909
    __ movdbl(dst_addr, src->as_xmm_double_reg());
910

911
#ifndef _LP64
912
  } else if (src->is_single_fpu()) {
913
    assert(src->fpu_regnr() == 0, "argument must be on TOS");
914
    Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
915
    if (pop_fpu_stack)     __ fstp_s (dst_addr);
916
    else                   __ fst_s  (dst_addr);
917

918
  } else if (src->is_double_fpu()) {
919
    assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
920
    Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
921
    if (pop_fpu_stack)     __ fstp_d (dst_addr);
922
    else                   __ fst_d  (dst_addr);
923
#endif // !_LP64
924

925
  } else {
926
    ShouldNotReachHere();
927
  }
928
}
929

930

931
void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
932
  LIR_Address* to_addr = dest->as_address_ptr();
933
  PatchingStub* patch = nullptr;
934
  Register compressed_src = rscratch1;
935

936
  if (is_reference_type(type)) {
937
    __ verify_oop(src->as_register());
938
#ifdef _LP64
939
    if (UseCompressedOops && !wide) {
940
      __ movptr(compressed_src, src->as_register());
941
      __ encode_heap_oop(compressed_src);
942
      if (patch_code != lir_patch_none) {
943
        info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
944
      }
945
    }
946
#endif
947
  }
948

949
  if (patch_code != lir_patch_none) {
950
    patch = new PatchingStub(_masm, PatchingStub::access_field_id);
951
    Address toa = as_Address(to_addr);
952
    assert(toa.disp() != 0, "must have");
953
  }
954

955
  int null_check_here = code_offset();
956
  switch (type) {
957
    case T_FLOAT: {
958
#ifdef _LP64
959
      assert(src->is_single_xmm(), "not a float");
960
      __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
961
#else
962
      if (src->is_single_xmm()) {
963
        __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
964
      } else {
965
        assert(src->is_single_fpu(), "must be");
966
        assert(src->fpu_regnr() == 0, "argument must be on TOS");
967
        if (pop_fpu_stack)      __ fstp_s(as_Address(to_addr));
968
        else                    __ fst_s (as_Address(to_addr));
969
      }
970
#endif // _LP64
971
      break;
972
    }
973

974
    case T_DOUBLE: {
975
#ifdef _LP64
976
      assert(src->is_double_xmm(), "not a double");
977
      __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
978
#else
979
      if (src->is_double_xmm()) {
980
        __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
981
      } else {
982
        assert(src->is_double_fpu(), "must be");
983
        assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
984
        if (pop_fpu_stack)      __ fstp_d(as_Address(to_addr));
985
        else                    __ fst_d (as_Address(to_addr));
986
      }
987
#endif // _LP64
988
      break;
989
    }
990

991
    case T_ARRAY:   // fall through
992
    case T_OBJECT:  // fall through
993
      if (UseCompressedOops && !wide) {
994
        __ movl(as_Address(to_addr), compressed_src);
995
      } else {
996
        __ movptr(as_Address(to_addr), src->as_register());
997
      }
998
      break;
999
    case T_METADATA:
1000
      // We get here to store a method pointer to the stack to pass to
1001
      // a dtrace runtime call. This can't work on 64 bit with
1002
      // compressed klass ptrs: T_METADATA can be a compressed klass
1003
      // ptr or a 64 bit method pointer.
1004
      LP64_ONLY(ShouldNotReachHere());
1005
      __ movptr(as_Address(to_addr), src->as_register());
1006
      break;
1007
    case T_ADDRESS:
1008
      __ movptr(as_Address(to_addr), src->as_register());
1009
      break;
1010
    case T_INT:
1011
      __ movl(as_Address(to_addr), src->as_register());
1012
      break;
1013

1014
    case T_LONG: {
1015
      Register from_lo = src->as_register_lo();
1016
      Register from_hi = src->as_register_hi();
1017
#ifdef _LP64
1018
      __ movptr(as_Address_lo(to_addr), from_lo);
1019
#else
1020
      Register base = to_addr->base()->as_register();
1021
      Register index = noreg;
1022
      if (to_addr->index()->is_register()) {
1023
        index = to_addr->index()->as_register();
1024
      }
1025
      if (base == from_lo || index == from_lo) {
1026
        assert(base != from_hi, "can't be");
1027
        assert(index == noreg || (index != base && index != from_hi), "can't handle this");
1028
        __ movl(as_Address_hi(to_addr), from_hi);
1029
        if (patch != nullptr) {
1030
          patching_epilog(patch, lir_patch_high, base, info);
1031
          patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1032
          patch_code = lir_patch_low;
1033
        }
1034
        __ movl(as_Address_lo(to_addr), from_lo);
1035
      } else {
1036
        assert(index == noreg || (index != base && index != from_lo), "can't handle this");
1037
        __ movl(as_Address_lo(to_addr), from_lo);
1038
        if (patch != nullptr) {
1039
          patching_epilog(patch, lir_patch_low, base, info);
1040
          patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1041
          patch_code = lir_patch_high;
1042
        }
1043
        __ movl(as_Address_hi(to_addr), from_hi);
1044
      }
1045
#endif // _LP64
1046
      break;
1047
    }
1048

1049
    case T_BYTE:    // fall through
1050
    case T_BOOLEAN: {
1051
      Register src_reg = src->as_register();
1052
      Address dst_addr = as_Address(to_addr);
1053
      assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
1054
      __ movb(dst_addr, src_reg);
1055
      break;
1056
    }
1057

1058
    case T_CHAR:    // fall through
1059
    case T_SHORT:
1060
      __ movw(as_Address(to_addr), src->as_register());
1061
      break;
1062

1063
    default:
1064
      ShouldNotReachHere();
1065
  }
1066
  if (info != nullptr) {
1067
    add_debug_info_for_null_check(null_check_here, info);
1068
  }
1069

1070
  if (patch_code != lir_patch_none) {
1071
    patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
1072
  }
1073
}
1074

1075

1076
void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1077
  assert(src->is_stack(), "should not call otherwise");
1078
  assert(dest->is_register(), "should not call otherwise");
1079

1080
  if (dest->is_single_cpu()) {
1081
    if (is_reference_type(type)) {
1082
      __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1083
      __ verify_oop(dest->as_register());
1084
    } else if (type == T_METADATA || type == T_ADDRESS) {
1085
      __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1086
    } else {
1087
      __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1088
    }
1089

1090
  } else if (dest->is_double_cpu()) {
1091
    Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
1092
    Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
1093
    __ movptr(dest->as_register_lo(), src_addr_LO);
1094
    NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI));
1095

1096
  } else if (dest->is_single_xmm()) {
1097
    Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1098
    __ movflt(dest->as_xmm_float_reg(), src_addr);
1099

1100
  } else if (dest->is_double_xmm()) {
1101
    Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1102
    __ movdbl(dest->as_xmm_double_reg(), src_addr);
1103

1104
#ifndef _LP64
1105
  } else if (dest->is_single_fpu()) {
1106
    assert(dest->fpu_regnr() == 0, "dest must be TOS");
1107
    Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1108
    __ fld_s(src_addr);
1109

1110
  } else if (dest->is_double_fpu()) {
1111
    assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1112
    Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1113
    __ fld_d(src_addr);
1114
#endif // _LP64
1115

1116
  } else {
1117
    ShouldNotReachHere();
1118
  }
1119
}
1120

1121

1122
void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1123
  if (src->is_single_stack()) {
1124
    if (is_reference_type(type)) {
1125
      __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
1126
      __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
1127
    } else {
1128
#ifndef _LP64
1129
      __ pushl(frame_map()->address_for_slot(src ->single_stack_ix()));
1130
      __ popl (frame_map()->address_for_slot(dest->single_stack_ix()));
1131
#else
1132
      //no pushl on 64bits
1133
      __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
1134
      __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
1135
#endif
1136
    }
1137

1138
  } else if (src->is_double_stack()) {
1139
#ifdef _LP64
1140
    __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
1141
    __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
1142
#else
1143
    __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0));
1144
    // push and pop the part at src + wordSize, adding wordSize for the previous push
1145
    __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize));
1146
    __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize));
1147
    __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0));
1148
#endif // _LP64
1149

1150
  } else {
1151
    ShouldNotReachHere();
1152
  }
1153
}
1154

1155

1156
void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
1157
  assert(src->is_address(), "should not call otherwise");
1158
  assert(dest->is_register(), "should not call otherwise");
1159

1160
  LIR_Address* addr = src->as_address_ptr();
1161
  Address from_addr = as_Address(addr);
1162

1163
  if (addr->base()->type() == T_OBJECT) {
1164
    __ verify_oop(addr->base()->as_pointer_register());
1165
  }
1166

1167
  switch (type) {
1168
    case T_BOOLEAN: // fall through
1169
    case T_BYTE:    // fall through
1170
    case T_CHAR:    // fall through
1171
    case T_SHORT:
1172
      if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1173
        // on pre P6 processors we may get partial register stalls
1174
        // so blow away the value of to_rinfo before loading a
1175
        // partial word into it.  Do it here so that it precedes
1176
        // the potential patch point below.
1177
        __ xorptr(dest->as_register(), dest->as_register());
1178
      }
1179
      break;
1180
   default:
1181
     break;
1182
  }
1183

1184
  PatchingStub* patch = nullptr;
1185
  if (patch_code != lir_patch_none) {
1186
    patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1187
    assert(from_addr.disp() != 0, "must have");
1188
  }
1189
  if (info != nullptr) {
1190
    add_debug_info_for_null_check_here(info);
1191
  }
1192

1193
  switch (type) {
1194
    case T_FLOAT: {
1195
      if (dest->is_single_xmm()) {
1196
        __ movflt(dest->as_xmm_float_reg(), from_addr);
1197
      } else {
1198
#ifndef _LP64
1199
        assert(dest->is_single_fpu(), "must be");
1200
        assert(dest->fpu_regnr() == 0, "dest must be TOS");
1201
        __ fld_s(from_addr);
1202
#else
1203
        ShouldNotReachHere();
1204
#endif // !LP64
1205
      }
1206
      break;
1207
    }
1208

1209
    case T_DOUBLE: {
1210
      if (dest->is_double_xmm()) {
1211
        __ movdbl(dest->as_xmm_double_reg(), from_addr);
1212
      } else {
1213
#ifndef _LP64
1214
        assert(dest->is_double_fpu(), "must be");
1215
        assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1216
        __ fld_d(from_addr);
1217
#else
1218
        ShouldNotReachHere();
1219
#endif // !LP64
1220
      }
1221
      break;
1222
    }
1223

1224
    case T_OBJECT:  // fall through
1225
    case T_ARRAY:   // fall through
1226
      if (UseCompressedOops && !wide) {
1227
        __ movl(dest->as_register(), from_addr);
1228
      } else {
1229
        __ movptr(dest->as_register(), from_addr);
1230
      }
1231
      break;
1232

1233
    case T_ADDRESS:
1234
      __ movptr(dest->as_register(), from_addr);
1235
      break;
1236
    case T_INT:
1237
      __ movl(dest->as_register(), from_addr);
1238
      break;
1239

1240
    case T_LONG: {
1241
      Register to_lo = dest->as_register_lo();
1242
      Register to_hi = dest->as_register_hi();
1243
#ifdef _LP64
1244
      __ movptr(to_lo, as_Address_lo(addr));
1245
#else
1246
      Register base = addr->base()->as_register();
1247
      Register index = noreg;
1248
      if (addr->index()->is_register()) {
1249
        index = addr->index()->as_register();
1250
      }
1251
      if ((base == to_lo && index == to_hi) ||
1252
          (base == to_hi && index == to_lo)) {
1253
        // addresses with 2 registers are only formed as a result of
1254
        // array access so this code will never have to deal with
1255
        // patches or null checks.
1256
        assert(info == nullptr && patch == nullptr, "must be");
1257
        __ lea(to_hi, as_Address(addr));
1258
        __ movl(to_lo, Address(to_hi, 0));
1259
        __ movl(to_hi, Address(to_hi, BytesPerWord));
1260
      } else if (base == to_lo || index == to_lo) {
1261
        assert(base != to_hi, "can't be");
1262
        assert(index == noreg || (index != base && index != to_hi), "can't handle this");
1263
        __ movl(to_hi, as_Address_hi(addr));
1264
        if (patch != nullptr) {
1265
          patching_epilog(patch, lir_patch_high, base, info);
1266
          patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1267
          patch_code = lir_patch_low;
1268
        }
1269
        __ movl(to_lo, as_Address_lo(addr));
1270
      } else {
1271
        assert(index == noreg || (index != base && index != to_lo), "can't handle this");
1272
        __ movl(to_lo, as_Address_lo(addr));
1273
        if (patch != nullptr) {
1274
          patching_epilog(patch, lir_patch_low, base, info);
1275
          patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1276
          patch_code = lir_patch_high;
1277
        }
1278
        __ movl(to_hi, as_Address_hi(addr));
1279
      }
1280
#endif // _LP64
1281
      break;
1282
    }
1283

1284
    case T_BOOLEAN: // fall through
1285
    case T_BYTE: {
1286
      Register dest_reg = dest->as_register();
1287
      assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1288
      if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1289
        __ movsbl(dest_reg, from_addr);
1290
      } else {
1291
        __ movb(dest_reg, from_addr);
1292
        __ shll(dest_reg, 24);
1293
        __ sarl(dest_reg, 24);
1294
      }
1295
      break;
1296
    }
1297

1298
    case T_CHAR: {
1299
      Register dest_reg = dest->as_register();
1300
      assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1301
      if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1302
        __ movzwl(dest_reg, from_addr);
1303
      } else {
1304
        __ movw(dest_reg, from_addr);
1305
      }
1306
      break;
1307
    }
1308

1309
    case T_SHORT: {
1310
      Register dest_reg = dest->as_register();
1311
      if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1312
        __ movswl(dest_reg, from_addr);
1313
      } else {
1314
        __ movw(dest_reg, from_addr);
1315
        __ shll(dest_reg, 16);
1316
        __ sarl(dest_reg, 16);
1317
      }
1318
      break;
1319
    }
1320

1321
    default:
1322
      ShouldNotReachHere();
1323
  }
1324

1325
  if (patch != nullptr) {
1326
    patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1327
  }
1328

1329
  if (is_reference_type(type)) {
1330
#ifdef _LP64
1331
    if (UseCompressedOops && !wide) {
1332
      __ decode_heap_oop(dest->as_register());
1333
    }
1334
#endif
1335

1336
    if (!(UseZGC && !ZGenerational)) {
1337
      // Load barrier has not yet been applied, so ZGC can't verify the oop here
1338
      __ verify_oop(dest->as_register());
1339
    }
1340
  }
1341
}
1342

1343

1344
NEEDS_CLEANUP; // This could be static?
1345
Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1346
  int elem_size = type2aelembytes(type);
1347
  switch (elem_size) {
1348
    case 1: return Address::times_1;
1349
    case 2: return Address::times_2;
1350
    case 4: return Address::times_4;
1351
    case 8: return Address::times_8;
1352
  }
1353
  ShouldNotReachHere();
1354
  return Address::no_scale;
1355
}
1356

1357

1358
void LIR_Assembler::emit_op3(LIR_Op3* op) {
1359
  switch (op->code()) {
1360
    case lir_idiv:
1361
    case lir_irem:
1362
      arithmetic_idiv(op->code(),
1363
                      op->in_opr1(),
1364
                      op->in_opr2(),
1365
                      op->in_opr3(),
1366
                      op->result_opr(),
1367
                      op->info());
1368
      break;
1369
    case lir_fmad:
1370
      __ fmad(op->result_opr()->as_xmm_double_reg(),
1371
              op->in_opr1()->as_xmm_double_reg(),
1372
              op->in_opr2()->as_xmm_double_reg(),
1373
              op->in_opr3()->as_xmm_double_reg());
1374
      break;
1375
    case lir_fmaf:
1376
      __ fmaf(op->result_opr()->as_xmm_float_reg(),
1377
              op->in_opr1()->as_xmm_float_reg(),
1378
              op->in_opr2()->as_xmm_float_reg(),
1379
              op->in_opr3()->as_xmm_float_reg());
1380
      break;
1381
    default:      ShouldNotReachHere(); break;
1382
  }
1383
}
1384

1385
void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1386
#ifdef ASSERT
1387
  assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1388
  if (op->block() != nullptr)  _branch_target_blocks.append(op->block());
1389
  if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1390
#endif
1391

1392
  if (op->cond() == lir_cond_always) {
1393
    if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1394
    __ jmp (*(op->label()));
1395
  } else {
1396
    Assembler::Condition acond = Assembler::zero;
1397
    if (op->code() == lir_cond_float_branch) {
1398
      assert(op->ublock() != nullptr, "must have unordered successor");
1399
      __ jcc(Assembler::parity, *(op->ublock()->label()));
1400
      switch(op->cond()) {
1401
        case lir_cond_equal:        acond = Assembler::equal;      break;
1402
        case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1403
        case lir_cond_less:         acond = Assembler::below;      break;
1404
        case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1405
        case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1406
        case lir_cond_greater:      acond = Assembler::above;      break;
1407
        default:                         ShouldNotReachHere();
1408
      }
1409
    } else {
1410
      switch (op->cond()) {
1411
        case lir_cond_equal:        acond = Assembler::equal;       break;
1412
        case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1413
        case lir_cond_less:         acond = Assembler::less;        break;
1414
        case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1415
        case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1416
        case lir_cond_greater:      acond = Assembler::greater;     break;
1417
        case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1418
        case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1419
        default:                         ShouldNotReachHere();
1420
      }
1421
    }
1422
    __ jcc(acond,*(op->label()));
1423
  }
1424
}
1425

1426
void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1427
  LIR_Opr src  = op->in_opr();
1428
  LIR_Opr dest = op->result_opr();
1429

1430
  switch (op->bytecode()) {
1431
    case Bytecodes::_i2l:
1432
#ifdef _LP64
1433
      __ movl2ptr(dest->as_register_lo(), src->as_register());
1434
#else
1435
      move_regs(src->as_register(), dest->as_register_lo());
1436
      move_regs(src->as_register(), dest->as_register_hi());
1437
      __ sarl(dest->as_register_hi(), 31);
1438
#endif // LP64
1439
      break;
1440

1441
    case Bytecodes::_l2i:
1442
#ifdef _LP64
1443
      __ movl(dest->as_register(), src->as_register_lo());
1444
#else
1445
      move_regs(src->as_register_lo(), dest->as_register());
1446
#endif
1447
      break;
1448

1449
    case Bytecodes::_i2b:
1450
      move_regs(src->as_register(), dest->as_register());
1451
      __ sign_extend_byte(dest->as_register());
1452
      break;
1453

1454
    case Bytecodes::_i2c:
1455
      move_regs(src->as_register(), dest->as_register());
1456
      __ andl(dest->as_register(), 0xFFFF);
1457
      break;
1458

1459
    case Bytecodes::_i2s:
1460
      move_regs(src->as_register(), dest->as_register());
1461
      __ sign_extend_short(dest->as_register());
1462
      break;
1463

1464

1465
#ifdef _LP64
1466
    case Bytecodes::_f2d:
1467
      __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1468
      break;
1469

1470
    case Bytecodes::_d2f:
1471
      __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1472
      break;
1473

1474
    case Bytecodes::_i2f:
1475
      __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1476
      break;
1477

1478
    case Bytecodes::_i2d:
1479
      __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1480
      break;
1481

1482
    case Bytecodes::_l2f:
1483
      __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1484
      break;
1485

1486
    case Bytecodes::_l2d:
1487
      __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1488
      break;
1489

1490
    case Bytecodes::_f2i:
1491
      __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1492
      break;
1493

1494
    case Bytecodes::_d2i:
1495
      __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1496
      break;
1497

1498
    case Bytecodes::_f2l:
1499
      __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1500
      break;
1501

1502
    case Bytecodes::_d2l:
1503
      __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1504
      break;
1505
#else
1506
    case Bytecodes::_f2d:
1507
    case Bytecodes::_d2f:
1508
      if (dest->is_single_xmm()) {
1509
        __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1510
      } else if (dest->is_double_xmm()) {
1511
        __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1512
      } else {
1513
        assert(src->fpu() == dest->fpu(), "register must be equal");
1514
        // do nothing (float result is rounded later through spilling)
1515
      }
1516
      break;
1517

1518
    case Bytecodes::_i2f:
1519
    case Bytecodes::_i2d:
1520
      if (dest->is_single_xmm()) {
1521
        __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1522
      } else if (dest->is_double_xmm()) {
1523
        __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1524
      } else {
1525
        assert(dest->fpu() == 0, "result must be on TOS");
1526
        __ movl(Address(rsp, 0), src->as_register());
1527
        __ fild_s(Address(rsp, 0));
1528
      }
1529
      break;
1530

1531
    case Bytecodes::_l2f:
1532
    case Bytecodes::_l2d:
1533
      assert(!dest->is_xmm_register(), "result in xmm register not supported (no SSE instruction present)");
1534
      assert(dest->fpu() == 0, "result must be on TOS");
1535
      __ movptr(Address(rsp, 0),          src->as_register_lo());
1536
      __ movl(Address(rsp, BytesPerWord), src->as_register_hi());
1537
      __ fild_d(Address(rsp, 0));
1538
      // float result is rounded later through spilling
1539
      break;
1540

1541
    case Bytecodes::_f2i:
1542
    case Bytecodes::_d2i:
1543
      if (src->is_single_xmm()) {
1544
        __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg());
1545
      } else if (src->is_double_xmm()) {
1546
        __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg());
1547
      } else {
1548
        assert(src->fpu() == 0, "input must be on TOS");
1549
        __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_trunc()));
1550
        __ fist_s(Address(rsp, 0));
1551
        __ movl(dest->as_register(), Address(rsp, 0));
1552
        __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std()));
1553
      }
1554
      // IA32 conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
1555
      assert(op->stub() != nullptr, "stub required");
1556
      __ cmpl(dest->as_register(), 0x80000000);
1557
      __ jcc(Assembler::equal, *op->stub()->entry());
1558
      __ bind(*op->stub()->continuation());
1559
      break;
1560

1561
    case Bytecodes::_f2l:
1562
    case Bytecodes::_d2l:
1563
      assert(!src->is_xmm_register(), "input in xmm register not supported (no SSE instruction present)");
1564
      assert(src->fpu() == 0, "input must be on TOS");
1565
      assert(dest == FrameMap::long0_opr, "runtime stub places result in these registers");
1566

1567
      // instruction sequence too long to inline it here
1568
      {
1569
        __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::fpu2long_stub_id)));
1570
      }
1571
      break;
1572
#endif // _LP64
1573

1574
    default: ShouldNotReachHere();
1575
  }
1576
}
1577

1578
void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1579
  if (op->init_check()) {
1580
    add_debug_info_for_null_check_here(op->stub()->info());
1581
    __ cmpb(Address(op->klass()->as_register(),
1582
                    InstanceKlass::init_state_offset()),
1583
                    InstanceKlass::fully_initialized);
1584
    __ jcc(Assembler::notEqual, *op->stub()->entry());
1585
  }
1586
  __ allocate_object(op->obj()->as_register(),
1587
                     op->tmp1()->as_register(),
1588
                     op->tmp2()->as_register(),
1589
                     op->header_size(),
1590
                     op->object_size(),
1591
                     op->klass()->as_register(),
1592
                     *op->stub()->entry());
1593
  __ bind(*op->stub()->continuation());
1594
}
1595

1596
void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1597
  Register len =  op->len()->as_register();
1598
  LP64_ONLY( __ movslq(len, len); )
1599

1600
  if (UseSlowPath ||
1601
      (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1602
      (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1603
    __ jmp(*op->stub()->entry());
1604
  } else {
1605
    Register tmp1 = op->tmp1()->as_register();
1606
    Register tmp2 = op->tmp2()->as_register();
1607
    Register tmp3 = op->tmp3()->as_register();
1608
    if (len == tmp1) {
1609
      tmp1 = tmp3;
1610
    } else if (len == tmp2) {
1611
      tmp2 = tmp3;
1612
    } else if (len == tmp3) {
1613
      // everything is ok
1614
    } else {
1615
      __ mov(tmp3, len);
1616
    }
1617
    __ allocate_array(op->obj()->as_register(),
1618
                      len,
1619
                      tmp1,
1620
                      tmp2,
1621
                      arrayOopDesc::base_offset_in_bytes(op->type()),
1622
                      array_element_size(op->type()),
1623
                      op->klass()->as_register(),
1624
                      *op->stub()->entry(),
1625
                      op->zero_array());
1626
  }
1627
  __ bind(*op->stub()->continuation());
1628
}
1629

1630
void LIR_Assembler::type_profile_helper(Register mdo,
1631
                                        ciMethodData *md, ciProfileData *data,
1632
                                        Register recv, Label* update_done) {
1633
  for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1634
    Label next_test;
1635
    // See if the receiver is receiver[n].
1636
    __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1637
    __ jccb(Assembler::notEqual, next_test);
1638
    Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1639
    __ addptr(data_addr, DataLayout::counter_increment);
1640
    __ jmp(*update_done);
1641
    __ bind(next_test);
1642
  }
1643

1644
  // Didn't find receiver; find next empty slot and fill it in
1645
  for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1646
    Label next_test;
1647
    Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1648
    __ cmpptr(recv_addr, NULL_WORD);
1649
    __ jccb(Assembler::notEqual, next_test);
1650
    __ movptr(recv_addr, recv);
1651
    __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1652
    __ jmp(*update_done);
1653
    __ bind(next_test);
1654
  }
1655
}
1656

1657
void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1658
  // we always need a stub for the failure case.
1659
  CodeStub* stub = op->stub();
1660
  Register obj = op->object()->as_register();
1661
  Register k_RInfo = op->tmp1()->as_register();
1662
  Register klass_RInfo = op->tmp2()->as_register();
1663
  Register dst = op->result_opr()->as_register();
1664
  ciKlass* k = op->klass();
1665
  Register Rtmp1 = noreg;
1666
  Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1667

1668
  // check if it needs to be profiled
1669
  ciMethodData* md = nullptr;
1670
  ciProfileData* data = nullptr;
1671

1672
  if (op->should_profile()) {
1673
    ciMethod* method = op->profiled_method();
1674
    assert(method != nullptr, "Should have method");
1675
    int bci = op->profiled_bci();
1676
    md = method->method_data_or_null();
1677
    assert(md != nullptr, "Sanity");
1678
    data = md->bci_to_data(bci);
1679
    assert(data != nullptr,                "need data for type check");
1680
    assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1681
  }
1682
  Label* success_target = success;
1683
  Label* failure_target = failure;
1684

1685
  if (obj == k_RInfo) {
1686
    k_RInfo = dst;
1687
  } else if (obj == klass_RInfo) {
1688
    klass_RInfo = dst;
1689
  }
1690
  if (k->is_loaded() && !UseCompressedClassPointers) {
1691
    select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1692
  } else {
1693
    Rtmp1 = op->tmp3()->as_register();
1694
    select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1695
  }
1696

1697
  assert_different_registers(obj, k_RInfo, klass_RInfo);
1698

1699
  __ testptr(obj, obj);
1700
  if (op->should_profile()) {
1701
    Label not_null;
1702
    Register mdo  = klass_RInfo;
1703
    __ mov_metadata(mdo, md->constant_encoding());
1704
    __ jccb(Assembler::notEqual, not_null);
1705
    // Object is null; update MDO and exit
1706
    Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1707
    int header_bits = BitData::null_seen_byte_constant();
1708
    __ orb(data_addr, header_bits);
1709
    __ jmp(*obj_is_null);
1710
    __ bind(not_null);
1711

1712
    Label update_done;
1713
    Register recv = k_RInfo;
1714
    __ load_klass(recv, obj, tmp_load_klass);
1715
    type_profile_helper(mdo, md, data, recv, &update_done);
1716

1717
    Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1718
    __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1719

1720
    __ bind(update_done);
1721
  } else {
1722
    __ jcc(Assembler::equal, *obj_is_null);
1723
  }
1724

1725
  if (!k->is_loaded()) {
1726
    klass2reg_with_patching(k_RInfo, op->info_for_patch());
1727
  } else {
1728
#ifdef _LP64
1729
    __ mov_metadata(k_RInfo, k->constant_encoding());
1730
#endif // _LP64
1731
  }
1732
  __ verify_oop(obj);
1733

1734
  if (op->fast_check()) {
1735
    // get object class
1736
    // not a safepoint as obj null check happens earlier
1737
#ifdef _LP64
1738
    if (UseCompressedClassPointers) {
1739
      __ load_klass(Rtmp1, obj, tmp_load_klass);
1740
      __ cmpptr(k_RInfo, Rtmp1);
1741
    } else {
1742
      __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1743
    }
1744
#else
1745
    if (k->is_loaded()) {
1746
      __ cmpklass(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
1747
    } else {
1748
      __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1749
    }
1750
#endif
1751
    __ jcc(Assembler::notEqual, *failure_target);
1752
    // successful cast, fall through to profile or jump
1753
  } else {
1754
    // get object class
1755
    // not a safepoint as obj null check happens earlier
1756
    __ load_klass(klass_RInfo, obj, tmp_load_klass);
1757
    if (k->is_loaded()) {
1758
      // See if we get an immediate positive hit
1759
#ifdef _LP64
1760
      __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1761
#else
1762
      __ cmpklass(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1763
#endif // _LP64
1764
      if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1765
        __ jcc(Assembler::notEqual, *failure_target);
1766
        // successful cast, fall through to profile or jump
1767
      } else {
1768
        // See if we get an immediate positive hit
1769
        __ jcc(Assembler::equal, *success_target);
1770
        // check for self
1771
#ifdef _LP64
1772
        __ cmpptr(klass_RInfo, k_RInfo);
1773
#else
1774
        __ cmpklass(klass_RInfo, k->constant_encoding());
1775
#endif // _LP64
1776
        __ jcc(Assembler::equal, *success_target);
1777

1778
        __ push(klass_RInfo);
1779
#ifdef _LP64
1780
        __ push(k_RInfo);
1781
#else
1782
        __ pushklass(k->constant_encoding(), noreg);
1783
#endif // _LP64
1784
        __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1785
        __ pop(klass_RInfo);
1786
        __ pop(klass_RInfo);
1787
        // result is a boolean
1788
        __ testl(klass_RInfo, klass_RInfo);
1789
        __ jcc(Assembler::equal, *failure_target);
1790
        // successful cast, fall through to profile or jump
1791
      }
1792
    } else {
1793
      // perform the fast part of the checking logic
1794
      __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1795
      // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1796
      __ push(klass_RInfo);
1797
      __ push(k_RInfo);
1798
      __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1799
      __ pop(klass_RInfo);
1800
      __ pop(k_RInfo);
1801
      // result is a boolean
1802
      __ testl(k_RInfo, k_RInfo);
1803
      __ jcc(Assembler::equal, *failure_target);
1804
      // successful cast, fall through to profile or jump
1805
    }
1806
  }
1807
  __ jmp(*success);
1808
}
1809

1810

1811
void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1812
  Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1813
  LIR_Code code = op->code();
1814
  if (code == lir_store_check) {
1815
    Register value = op->object()->as_register();
1816
    Register array = op->array()->as_register();
1817
    Register k_RInfo = op->tmp1()->as_register();
1818
    Register klass_RInfo = op->tmp2()->as_register();
1819
    Register Rtmp1 = op->tmp3()->as_register();
1820

1821
    CodeStub* stub = op->stub();
1822

1823
    // check if it needs to be profiled
1824
    ciMethodData* md = nullptr;
1825
    ciProfileData* data = nullptr;
1826

1827
    if (op->should_profile()) {
1828
      ciMethod* method = op->profiled_method();
1829
      assert(method != nullptr, "Should have method");
1830
      int bci = op->profiled_bci();
1831
      md = method->method_data_or_null();
1832
      assert(md != nullptr, "Sanity");
1833
      data = md->bci_to_data(bci);
1834
      assert(data != nullptr,                "need data for type check");
1835
      assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1836
    }
1837
    Label done;
1838
    Label* success_target = &done;
1839
    Label* failure_target = stub->entry();
1840

1841
    __ testptr(value, value);
1842
    if (op->should_profile()) {
1843
      Label not_null;
1844
      Register mdo  = klass_RInfo;
1845
      __ mov_metadata(mdo, md->constant_encoding());
1846
      __ jccb(Assembler::notEqual, not_null);
1847
      // Object is null; update MDO and exit
1848
      Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1849
      int header_bits = BitData::null_seen_byte_constant();
1850
      __ orb(data_addr, header_bits);
1851
      __ jmp(done);
1852
      __ bind(not_null);
1853

1854
      Label update_done;
1855
      Register recv = k_RInfo;
1856
      __ load_klass(recv, value, tmp_load_klass);
1857
      type_profile_helper(mdo, md, data, recv, &update_done);
1858

1859
      Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1860
      __ addptr(counter_addr, DataLayout::counter_increment);
1861
      __ bind(update_done);
1862
    } else {
1863
      __ jcc(Assembler::equal, done);
1864
    }
1865

1866
    add_debug_info_for_null_check_here(op->info_for_exception());
1867
    __ load_klass(k_RInfo, array, tmp_load_klass);
1868
    __ load_klass(klass_RInfo, value, tmp_load_klass);
1869

1870
    // get instance klass (it's already uncompressed)
1871
    __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1872
    // perform the fast part of the checking logic
1873
    __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1874
    // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1875
    __ push(klass_RInfo);
1876
    __ push(k_RInfo);
1877
    __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1878
    __ pop(klass_RInfo);
1879
    __ pop(k_RInfo);
1880
    // result is a boolean
1881
    __ testl(k_RInfo, k_RInfo);
1882
    __ jcc(Assembler::equal, *failure_target);
1883
    // fall through to the success case
1884

1885
    __ bind(done);
1886
  } else
1887
    if (code == lir_checkcast) {
1888
      Register obj = op->object()->as_register();
1889
      Register dst = op->result_opr()->as_register();
1890
      Label success;
1891
      emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1892
      __ bind(success);
1893
      if (dst != obj) {
1894
        __ mov(dst, obj);
1895
      }
1896
    } else
1897
      if (code == lir_instanceof) {
1898
        Register obj = op->object()->as_register();
1899
        Register dst = op->result_opr()->as_register();
1900
        Label success, failure, done;
1901
        emit_typecheck_helper(op, &success, &failure, &failure);
1902
        __ bind(failure);
1903
        __ xorptr(dst, dst);
1904
        __ jmpb(done);
1905
        __ bind(success);
1906
        __ movptr(dst, 1);
1907
        __ bind(done);
1908
      } else {
1909
        ShouldNotReachHere();
1910
      }
1911

1912
}
1913

1914

1915
void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1916
  if (LP64_ONLY(false &&) op->code() == lir_cas_long) {
1917
    assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1918
    assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1919
    assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1920
    assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1921
    Register addr = op->addr()->as_register();
1922
    __ lock();
1923
    NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1924

1925
  } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1926
    NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1927
    Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1928
    Register newval = op->new_value()->as_register();
1929
    Register cmpval = op->cmp_value()->as_register();
1930
    assert(cmpval == rax, "wrong register");
1931
    assert(newval != noreg, "new val must be register");
1932
    assert(cmpval != newval, "cmp and new values must be in different registers");
1933
    assert(cmpval != addr, "cmp and addr must be in different registers");
1934
    assert(newval != addr, "new value and addr must be in different registers");
1935

1936
    if ( op->code() == lir_cas_obj) {
1937
#ifdef _LP64
1938
      if (UseCompressedOops) {
1939
        __ encode_heap_oop(cmpval);
1940
        __ mov(rscratch1, newval);
1941
        __ encode_heap_oop(rscratch1);
1942
        __ lock();
1943
        // cmpval (rax) is implicitly used by this instruction
1944
        __ cmpxchgl(rscratch1, Address(addr, 0));
1945
      } else
1946
#endif
1947
      {
1948
        __ lock();
1949
        __ cmpxchgptr(newval, Address(addr, 0));
1950
      }
1951
    } else {
1952
      assert(op->code() == lir_cas_int, "lir_cas_int expected");
1953
      __ lock();
1954
      __ cmpxchgl(newval, Address(addr, 0));
1955
    }
1956
#ifdef _LP64
1957
  } else if (op->code() == lir_cas_long) {
1958
    Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1959
    Register newval = op->new_value()->as_register_lo();
1960
    Register cmpval = op->cmp_value()->as_register_lo();
1961
    assert(cmpval == rax, "wrong register");
1962
    assert(newval != noreg, "new val must be register");
1963
    assert(cmpval != newval, "cmp and new values must be in different registers");
1964
    assert(cmpval != addr, "cmp and addr must be in different registers");
1965
    assert(newval != addr, "new value and addr must be in different registers");
1966
    __ lock();
1967
    __ cmpxchgq(newval, Address(addr, 0));
1968
#endif // _LP64
1969
  } else {
1970
    Unimplemented();
1971
  }
1972
}
1973

1974
void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1975
                          LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1976
  assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1977

1978
  Assembler::Condition acond, ncond;
1979
  switch (condition) {
1980
    case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1981
    case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1982
    case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1983
    case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1984
    case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1985
    case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1986
    case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1987
    case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1988
    default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1989
                                ShouldNotReachHere();
1990
  }
1991

1992
  if (opr1->is_cpu_register()) {
1993
    reg2reg(opr1, result);
1994
  } else if (opr1->is_stack()) {
1995
    stack2reg(opr1, result, result->type());
1996
  } else if (opr1->is_constant()) {
1997
    const2reg(opr1, result, lir_patch_none, nullptr);
1998
  } else {
1999
    ShouldNotReachHere();
2000
  }
2001

2002
  if (VM_Version::supports_cmov() && !opr2->is_constant()) {
2003
    // optimized version that does not require a branch
2004
    if (opr2->is_single_cpu()) {
2005
      assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
2006
      __ cmov(ncond, result->as_register(), opr2->as_register());
2007
    } else if (opr2->is_double_cpu()) {
2008
      assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2009
      assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2010
      __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
2011
      NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());)
2012
    } else if (opr2->is_single_stack()) {
2013
      __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
2014
    } else if (opr2->is_double_stack()) {
2015
      __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
2016
      NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));)
2017
    } else {
2018
      ShouldNotReachHere();
2019
    }
2020

2021
  } else {
2022
    Label skip;
2023
    __ jccb(acond, skip);
2024
    if (opr2->is_cpu_register()) {
2025
      reg2reg(opr2, result);
2026
    } else if (opr2->is_stack()) {
2027
      stack2reg(opr2, result, result->type());
2028
    } else if (opr2->is_constant()) {
2029
      const2reg(opr2, result, lir_patch_none, nullptr);
2030
    } else {
2031
      ShouldNotReachHere();
2032
    }
2033
    __ bind(skip);
2034
  }
2035
}
2036

2037

2038
void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
2039
  assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
2040

2041
  if (left->is_single_cpu()) {
2042
    assert(left == dest, "left and dest must be equal");
2043
    Register lreg = left->as_register();
2044

2045
    if (right->is_single_cpu()) {
2046
      // cpu register - cpu register
2047
      Register rreg = right->as_register();
2048
      switch (code) {
2049
        case lir_add: __ addl (lreg, rreg); break;
2050
        case lir_sub: __ subl (lreg, rreg); break;
2051
        case lir_mul: __ imull(lreg, rreg); break;
2052
        default:      ShouldNotReachHere();
2053
      }
2054

2055
    } else if (right->is_stack()) {
2056
      // cpu register - stack
2057
      Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2058
      switch (code) {
2059
        case lir_add: __ addl(lreg, raddr); break;
2060
        case lir_sub: __ subl(lreg, raddr); break;
2061
        default:      ShouldNotReachHere();
2062
      }
2063

2064
    } else if (right->is_constant()) {
2065
      // cpu register - constant
2066
      jint c = right->as_constant_ptr()->as_jint();
2067
      switch (code) {
2068
        case lir_add: {
2069
          __ incrementl(lreg, c);
2070
          break;
2071
        }
2072
        case lir_sub: {
2073
          __ decrementl(lreg, c);
2074
          break;
2075
        }
2076
        default: ShouldNotReachHere();
2077
      }
2078

2079
    } else {
2080
      ShouldNotReachHere();
2081
    }
2082

2083
  } else if (left->is_double_cpu()) {
2084
    assert(left == dest, "left and dest must be equal");
2085
    Register lreg_lo = left->as_register_lo();
2086
    Register lreg_hi = left->as_register_hi();
2087

2088
    if (right->is_double_cpu()) {
2089
      // cpu register - cpu register
2090
      Register rreg_lo = right->as_register_lo();
2091
      Register rreg_hi = right->as_register_hi();
2092
      NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi));
2093
      LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo));
2094
      switch (code) {
2095
        case lir_add:
2096
          __ addptr(lreg_lo, rreg_lo);
2097
          NOT_LP64(__ adcl(lreg_hi, rreg_hi));
2098
          break;
2099
        case lir_sub:
2100
          __ subptr(lreg_lo, rreg_lo);
2101
          NOT_LP64(__ sbbl(lreg_hi, rreg_hi));
2102
          break;
2103
        case lir_mul:
2104
#ifdef _LP64
2105
          __ imulq(lreg_lo, rreg_lo);
2106
#else
2107
          assert(lreg_lo == rax && lreg_hi == rdx, "must be");
2108
          __ imull(lreg_hi, rreg_lo);
2109
          __ imull(rreg_hi, lreg_lo);
2110
          __ addl (rreg_hi, lreg_hi);
2111
          __ mull (rreg_lo);
2112
          __ addl (lreg_hi, rreg_hi);
2113
#endif // _LP64
2114
          break;
2115
        default:
2116
          ShouldNotReachHere();
2117
      }
2118

2119
    } else if (right->is_constant()) {
2120
      // cpu register - constant
2121
#ifdef _LP64
2122
      jlong c = right->as_constant_ptr()->as_jlong_bits();
2123
      __ movptr(r10, (intptr_t) c);
2124
      switch (code) {
2125
        case lir_add:
2126
          __ addptr(lreg_lo, r10);
2127
          break;
2128
        case lir_sub:
2129
          __ subptr(lreg_lo, r10);
2130
          break;
2131
        default:
2132
          ShouldNotReachHere();
2133
      }
2134
#else
2135
      jint c_lo = right->as_constant_ptr()->as_jint_lo();
2136
      jint c_hi = right->as_constant_ptr()->as_jint_hi();
2137
      switch (code) {
2138
        case lir_add:
2139
          __ addptr(lreg_lo, c_lo);
2140
          __ adcl(lreg_hi, c_hi);
2141
          break;
2142
        case lir_sub:
2143
          __ subptr(lreg_lo, c_lo);
2144
          __ sbbl(lreg_hi, c_hi);
2145
          break;
2146
        default:
2147
          ShouldNotReachHere();
2148
      }
2149
#endif // _LP64
2150

2151
    } else {
2152
      ShouldNotReachHere();
2153
    }
2154

2155
  } else if (left->is_single_xmm()) {
2156
    assert(left == dest, "left and dest must be equal");
2157
    XMMRegister lreg = left->as_xmm_float_reg();
2158

2159
    if (right->is_single_xmm()) {
2160
      XMMRegister rreg = right->as_xmm_float_reg();
2161
      switch (code) {
2162
        case lir_add: __ addss(lreg, rreg);  break;
2163
        case lir_sub: __ subss(lreg, rreg);  break;
2164
        case lir_mul: __ mulss(lreg, rreg);  break;
2165
        case lir_div: __ divss(lreg, rreg);  break;
2166
        default: ShouldNotReachHere();
2167
      }
2168
    } else {
2169
      Address raddr;
2170
      if (right->is_single_stack()) {
2171
        raddr = frame_map()->address_for_slot(right->single_stack_ix());
2172
      } else if (right->is_constant()) {
2173
        // hack for now
2174
        raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
2175
      } else {
2176
        ShouldNotReachHere();
2177
      }
2178
      switch (code) {
2179
        case lir_add: __ addss(lreg, raddr);  break;
2180
        case lir_sub: __ subss(lreg, raddr);  break;
2181
        case lir_mul: __ mulss(lreg, raddr);  break;
2182
        case lir_div: __ divss(lreg, raddr);  break;
2183
        default: ShouldNotReachHere();
2184
      }
2185
    }
2186

2187
  } else if (left->is_double_xmm()) {
2188
    assert(left == dest, "left and dest must be equal");
2189

2190
    XMMRegister lreg = left->as_xmm_double_reg();
2191
    if (right->is_double_xmm()) {
2192
      XMMRegister rreg = right->as_xmm_double_reg();
2193
      switch (code) {
2194
        case lir_add: __ addsd(lreg, rreg);  break;
2195
        case lir_sub: __ subsd(lreg, rreg);  break;
2196
        case lir_mul: __ mulsd(lreg, rreg);  break;
2197
        case lir_div: __ divsd(lreg, rreg);  break;
2198
        default: ShouldNotReachHere();
2199
      }
2200
    } else {
2201
      Address raddr;
2202
      if (right->is_double_stack()) {
2203
        raddr = frame_map()->address_for_slot(right->double_stack_ix());
2204
      } else if (right->is_constant()) {
2205
        // hack for now
2206
        raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2207
      } else {
2208
        ShouldNotReachHere();
2209
      }
2210
      switch (code) {
2211
        case lir_add: __ addsd(lreg, raddr);  break;
2212
        case lir_sub: __ subsd(lreg, raddr);  break;
2213
        case lir_mul: __ mulsd(lreg, raddr);  break;
2214
        case lir_div: __ divsd(lreg, raddr);  break;
2215
        default: ShouldNotReachHere();
2216
      }
2217
    }
2218

2219
#ifndef _LP64
2220
  } else if (left->is_single_fpu()) {
2221
    assert(dest->is_single_fpu(),  "fpu stack allocation required");
2222

2223
    if (right->is_single_fpu()) {
2224
      arith_fpu_implementation(code, left->fpu_regnr(), right->fpu_regnr(), dest->fpu_regnr(), pop_fpu_stack);
2225

2226
    } else {
2227
      assert(left->fpu_regnr() == 0, "left must be on TOS");
2228
      assert(dest->fpu_regnr() == 0, "dest must be on TOS");
2229

2230
      Address raddr;
2231
      if (right->is_single_stack()) {
2232
        raddr = frame_map()->address_for_slot(right->single_stack_ix());
2233
      } else if (right->is_constant()) {
2234
        address const_addr = float_constant(right->as_jfloat());
2235
        assert(const_addr != nullptr, "incorrect float/double constant maintenance");
2236
        // hack for now
2237
        raddr = __ as_Address(InternalAddress(const_addr));
2238
      } else {
2239
        ShouldNotReachHere();
2240
      }
2241

2242
      switch (code) {
2243
        case lir_add: __ fadd_s(raddr); break;
2244
        case lir_sub: __ fsub_s(raddr); break;
2245
        case lir_mul: __ fmul_s(raddr); break;
2246
        case lir_div: __ fdiv_s(raddr); break;
2247
        default:      ShouldNotReachHere();
2248
      }
2249
    }
2250

2251
  } else if (left->is_double_fpu()) {
2252
    assert(dest->is_double_fpu(),  "fpu stack allocation required");
2253

2254
    if (code == lir_mul || code == lir_div) {
2255
      // Double values require special handling for strictfp mul/div on x86
2256
      __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias1()));
2257
      __ fmulp(left->fpu_regnrLo() + 1);
2258
    }
2259

2260
    if (right->is_double_fpu()) {
2261
      arith_fpu_implementation(code, left->fpu_regnrLo(), right->fpu_regnrLo(), dest->fpu_regnrLo(), pop_fpu_stack);
2262

2263
    } else {
2264
      assert(left->fpu_regnrLo() == 0, "left must be on TOS");
2265
      assert(dest->fpu_regnrLo() == 0, "dest must be on TOS");
2266

2267
      Address raddr;
2268
      if (right->is_double_stack()) {
2269
        raddr = frame_map()->address_for_slot(right->double_stack_ix());
2270
      } else if (right->is_constant()) {
2271
        // hack for now
2272
        raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2273
      } else {
2274
        ShouldNotReachHere();
2275
      }
2276

2277
      switch (code) {
2278
        case lir_add: __ fadd_d(raddr); break;
2279
        case lir_sub: __ fsub_d(raddr); break;
2280
        case lir_mul: __ fmul_d(raddr); break;
2281
        case lir_div: __ fdiv_d(raddr); break;
2282
        default: ShouldNotReachHere();
2283
      }
2284
    }
2285

2286
    if (code == lir_mul || code == lir_div) {
2287
      // Double values require special handling for strictfp mul/div on x86
2288
      __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias2()));
2289
      __ fmulp(dest->fpu_regnrLo() + 1);
2290
    }
2291
#endif // !_LP64
2292

2293
  } else if (left->is_single_stack() || left->is_address()) {
2294
    assert(left == dest, "left and dest must be equal");
2295

2296
    Address laddr;
2297
    if (left->is_single_stack()) {
2298
      laddr = frame_map()->address_for_slot(left->single_stack_ix());
2299
    } else if (left->is_address()) {
2300
      laddr = as_Address(left->as_address_ptr());
2301
    } else {
2302
      ShouldNotReachHere();
2303
    }
2304

2305
    if (right->is_single_cpu()) {
2306
      Register rreg = right->as_register();
2307
      switch (code) {
2308
        case lir_add: __ addl(laddr, rreg); break;
2309
        case lir_sub: __ subl(laddr, rreg); break;
2310
        default:      ShouldNotReachHere();
2311
      }
2312
    } else if (right->is_constant()) {
2313
      jint c = right->as_constant_ptr()->as_jint();
2314
      switch (code) {
2315
        case lir_add: {
2316
          __ incrementl(laddr, c);
2317
          break;
2318
        }
2319
        case lir_sub: {
2320
          __ decrementl(laddr, c);
2321
          break;
2322
        }
2323
        default: ShouldNotReachHere();
2324
      }
2325
    } else {
2326
      ShouldNotReachHere();
2327
    }
2328

2329
  } else {
2330
    ShouldNotReachHere();
2331
  }
2332
}
2333

2334
#ifndef _LP64
2335
void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) {
2336
  assert(pop_fpu_stack  || (left_index     == dest_index || right_index     == dest_index), "invalid LIR");
2337
  assert(!pop_fpu_stack || (left_index - 1 == dest_index || right_index - 1 == dest_index), "invalid LIR");
2338
  assert(left_index == 0 || right_index == 0, "either must be on top of stack");
2339

2340
  bool left_is_tos = (left_index == 0);
2341
  bool dest_is_tos = (dest_index == 0);
2342
  int non_tos_index = (left_is_tos ? right_index : left_index);
2343

2344
  switch (code) {
2345
    case lir_add:
2346
      if (pop_fpu_stack)       __ faddp(non_tos_index);
2347
      else if (dest_is_tos)    __ fadd (non_tos_index);
2348
      else                     __ fadda(non_tos_index);
2349
      break;
2350

2351
    case lir_sub:
2352
      if (left_is_tos) {
2353
        if (pop_fpu_stack)     __ fsubrp(non_tos_index);
2354
        else if (dest_is_tos)  __ fsub  (non_tos_index);
2355
        else                   __ fsubra(non_tos_index);
2356
      } else {
2357
        if (pop_fpu_stack)     __ fsubp (non_tos_index);
2358
        else if (dest_is_tos)  __ fsubr (non_tos_index);
2359
        else                   __ fsuba (non_tos_index);
2360
      }
2361
      break;
2362

2363
    case lir_mul:
2364
      if (pop_fpu_stack)       __ fmulp(non_tos_index);
2365
      else if (dest_is_tos)    __ fmul (non_tos_index);
2366
      else                     __ fmula(non_tos_index);
2367
      break;
2368

2369
    case lir_div:
2370
      if (left_is_tos) {
2371
        if (pop_fpu_stack)     __ fdivrp(non_tos_index);
2372
        else if (dest_is_tos)  __ fdiv  (non_tos_index);
2373
        else                   __ fdivra(non_tos_index);
2374
      } else {
2375
        if (pop_fpu_stack)     __ fdivp (non_tos_index);
2376
        else if (dest_is_tos)  __ fdivr (non_tos_index);
2377
        else                   __ fdiva (non_tos_index);
2378
      }
2379
      break;
2380

2381
    case lir_rem:
2382
      assert(left_is_tos && dest_is_tos && right_index == 1, "must be guaranteed by FPU stack allocation");
2383
      __ fremr(noreg);
2384
      break;
2385

2386
    default:
2387
      ShouldNotReachHere();
2388
  }
2389
}
2390
#endif // _LP64
2391

2392

2393
void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2394
  if (value->is_double_xmm()) {
2395
    switch(code) {
2396
      case lir_abs :
2397
        {
2398
#ifdef _LP64
2399
          if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
2400
            assert(tmp->is_valid(), "need temporary");
2401
            __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2);
2402
          } else
2403
#endif
2404
          {
2405
            if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2406
              __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2407
            }
2408
            assert(!tmp->is_valid(), "do not need temporary");
2409
            __ andpd(dest->as_xmm_double_reg(),
2410
                     ExternalAddress((address)double_signmask_pool),
2411
                     rscratch1);
2412
          }
2413
        }
2414
        break;
2415

2416
      case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2417
      // all other intrinsics are not available in the SSE instruction set, so FPU is used
2418
      default      : ShouldNotReachHere();
2419
    }
2420

2421
#ifndef _LP64
2422
  } else if (value->is_double_fpu()) {
2423
    assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2424
    switch(code) {
2425
      case lir_abs   : __ fabs() ; break;
2426
      case lir_sqrt  : __ fsqrt(); break;
2427
      default      : ShouldNotReachHere();
2428
    }
2429
#endif // !_LP64
2430
  } else if (code == lir_f2hf) {
2431
    __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
2432
  } else if (code == lir_hf2f) {
2433
    __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
2434
  } else {
2435
    Unimplemented();
2436
  }
2437
}
2438

2439
void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2440
  // assert(left->destroys_register(), "check");
2441
  if (left->is_single_cpu()) {
2442
    Register reg = left->as_register();
2443
    if (right->is_constant()) {
2444
      int val = right->as_constant_ptr()->as_jint();
2445
      switch (code) {
2446
        case lir_logic_and: __ andl (reg, val); break;
2447
        case lir_logic_or:  __ orl  (reg, val); break;
2448
        case lir_logic_xor: __ xorl (reg, val); break;
2449
        default: ShouldNotReachHere();
2450
      }
2451
    } else if (right->is_stack()) {
2452
      // added support for stack operands
2453
      Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2454
      switch (code) {
2455
        case lir_logic_and: __ andl (reg, raddr); break;
2456
        case lir_logic_or:  __ orl  (reg, raddr); break;
2457
        case lir_logic_xor: __ xorl (reg, raddr); break;
2458
        default: ShouldNotReachHere();
2459
      }
2460
    } else {
2461
      Register rright = right->as_register();
2462
      switch (code) {
2463
        case lir_logic_and: __ andptr (reg, rright); break;
2464
        case lir_logic_or : __ orptr  (reg, rright); break;
2465
        case lir_logic_xor: __ xorptr (reg, rright); break;
2466
        default: ShouldNotReachHere();
2467
      }
2468
    }
2469
    move_regs(reg, dst->as_register());
2470
  } else {
2471
    Register l_lo = left->as_register_lo();
2472
    Register l_hi = left->as_register_hi();
2473
    if (right->is_constant()) {
2474
#ifdef _LP64
2475
      __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2476
      switch (code) {
2477
        case lir_logic_and:
2478
          __ andq(l_lo, rscratch1);
2479
          break;
2480
        case lir_logic_or:
2481
          __ orq(l_lo, rscratch1);
2482
          break;
2483
        case lir_logic_xor:
2484
          __ xorq(l_lo, rscratch1);
2485
          break;
2486
        default: ShouldNotReachHere();
2487
      }
2488
#else
2489
      int r_lo = right->as_constant_ptr()->as_jint_lo();
2490
      int r_hi = right->as_constant_ptr()->as_jint_hi();
2491
      switch (code) {
2492
        case lir_logic_and:
2493
          __ andl(l_lo, r_lo);
2494
          __ andl(l_hi, r_hi);
2495
          break;
2496
        case lir_logic_or:
2497
          __ orl(l_lo, r_lo);
2498
          __ orl(l_hi, r_hi);
2499
          break;
2500
        case lir_logic_xor:
2501
          __ xorl(l_lo, r_lo);
2502
          __ xorl(l_hi, r_hi);
2503
          break;
2504
        default: ShouldNotReachHere();
2505
      }
2506
#endif // _LP64
2507
    } else {
2508
#ifdef _LP64
2509
      Register r_lo;
2510
      if (is_reference_type(right->type())) {
2511
        r_lo = right->as_register();
2512
      } else {
2513
        r_lo = right->as_register_lo();
2514
      }
2515
#else
2516
      Register r_lo = right->as_register_lo();
2517
      Register r_hi = right->as_register_hi();
2518
      assert(l_lo != r_hi, "overwriting registers");
2519
#endif
2520
      switch (code) {
2521
        case lir_logic_and:
2522
          __ andptr(l_lo, r_lo);
2523
          NOT_LP64(__ andptr(l_hi, r_hi);)
2524
          break;
2525
        case lir_logic_or:
2526
          __ orptr(l_lo, r_lo);
2527
          NOT_LP64(__ orptr(l_hi, r_hi);)
2528
          break;
2529
        case lir_logic_xor:
2530
          __ xorptr(l_lo, r_lo);
2531
          NOT_LP64(__ xorptr(l_hi, r_hi);)
2532
          break;
2533
        default: ShouldNotReachHere();
2534
      }
2535
    }
2536

2537
    Register dst_lo = dst->as_register_lo();
2538
    Register dst_hi = dst->as_register_hi();
2539

2540
#ifdef _LP64
2541
    move_regs(l_lo, dst_lo);
2542
#else
2543
    if (dst_lo == l_hi) {
2544
      assert(dst_hi != l_lo, "overwriting registers");
2545
      move_regs(l_hi, dst_hi);
2546
      move_regs(l_lo, dst_lo);
2547
    } else {
2548
      assert(dst_lo != l_hi, "overwriting registers");
2549
      move_regs(l_lo, dst_lo);
2550
      move_regs(l_hi, dst_hi);
2551
    }
2552
#endif // _LP64
2553
  }
2554
}
2555

2556

2557
// we assume that rax, and rdx can be overwritten
2558
void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2559

2560
  assert(left->is_single_cpu(),   "left must be register");
2561
  assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2562
  assert(result->is_single_cpu(), "result must be register");
2563

2564
  //  assert(left->destroys_register(), "check");
2565
  //  assert(right->destroys_register(), "check");
2566

2567
  Register lreg = left->as_register();
2568
  Register dreg = result->as_register();
2569

2570
  if (right->is_constant()) {
2571
    jint divisor = right->as_constant_ptr()->as_jint();
2572
    assert(divisor > 0 && is_power_of_2(divisor), "must be");
2573
    if (code == lir_idiv) {
2574
      assert(lreg == rax, "must be rax,");
2575
      assert(temp->as_register() == rdx, "tmp register must be rdx");
2576
      __ cdql(); // sign extend into rdx:rax
2577
      if (divisor == 2) {
2578
        __ subl(lreg, rdx);
2579
      } else {
2580
        __ andl(rdx, divisor - 1);
2581
        __ addl(lreg, rdx);
2582
      }
2583
      __ sarl(lreg, log2i_exact(divisor));
2584
      move_regs(lreg, dreg);
2585
    } else if (code == lir_irem) {
2586
      Label done;
2587
      __ mov(dreg, lreg);
2588
      __ andl(dreg, 0x80000000 | (divisor - 1));
2589
      __ jcc(Assembler::positive, done);
2590
      __ decrement(dreg);
2591
      __ orl(dreg, ~(divisor - 1));
2592
      __ increment(dreg);
2593
      __ bind(done);
2594
    } else {
2595
      ShouldNotReachHere();
2596
    }
2597
  } else {
2598
    Register rreg = right->as_register();
2599
    assert(lreg == rax, "left register must be rax,");
2600
    assert(rreg != rdx, "right register must not be rdx");
2601
    assert(temp->as_register() == rdx, "tmp register must be rdx");
2602

2603
    move_regs(lreg, rax);
2604

2605
    int idivl_offset = __ corrected_idivl(rreg);
2606
    if (ImplicitDiv0Checks) {
2607
      add_debug_info_for_div0(idivl_offset, info);
2608
    }
2609
    if (code == lir_irem) {
2610
      move_regs(rdx, dreg); // result is in rdx
2611
    } else {
2612
      move_regs(rax, dreg);
2613
    }
2614
  }
2615
}
2616

2617

2618
void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2619
  if (opr1->is_single_cpu()) {
2620
    Register reg1 = opr1->as_register();
2621
    if (opr2->is_single_cpu()) {
2622
      // cpu register - cpu register
2623
      if (is_reference_type(opr1->type())) {
2624
        __ cmpoop(reg1, opr2->as_register());
2625
      } else {
2626
        assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2627
        __ cmpl(reg1, opr2->as_register());
2628
      }
2629
    } else if (opr2->is_stack()) {
2630
      // cpu register - stack
2631
      if (is_reference_type(opr1->type())) {
2632
        __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2633
      } else {
2634
        __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2635
      }
2636
    } else if (opr2->is_constant()) {
2637
      // cpu register - constant
2638
      LIR_Const* c = opr2->as_constant_ptr();
2639
      if (c->type() == T_INT) {
2640
        jint i = c->as_jint();
2641
        if (i == 0) {
2642
          __ testl(reg1, reg1);
2643
        } else {
2644
          __ cmpl(reg1, i);
2645
        }
2646
      } else if (c->type() == T_METADATA) {
2647
        // All we need for now is a comparison with null for equality.
2648
        assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2649
        Metadata* m = c->as_metadata();
2650
        if (m == nullptr) {
2651
          __ testptr(reg1, reg1);
2652
        } else {
2653
          ShouldNotReachHere();
2654
        }
2655
      } else if (is_reference_type(c->type())) {
2656
        // In 64bit oops are single register
2657
        jobject o = c->as_jobject();
2658
        if (o == nullptr) {
2659
          __ testptr(reg1, reg1);
2660
        } else {
2661
          __ cmpoop(reg1, o, rscratch1);
2662
        }
2663
      } else {
2664
        fatal("unexpected type: %s", basictype_to_str(c->type()));
2665
      }
2666
      // cpu register - address
2667
    } else if (opr2->is_address()) {
2668
      if (op->info() != nullptr) {
2669
        add_debug_info_for_null_check_here(op->info());
2670
      }
2671
      __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2672
    } else {
2673
      ShouldNotReachHere();
2674
    }
2675

2676
  } else if(opr1->is_double_cpu()) {
2677
    Register xlo = opr1->as_register_lo();
2678
    Register xhi = opr1->as_register_hi();
2679
    if (opr2->is_double_cpu()) {
2680
#ifdef _LP64
2681
      __ cmpptr(xlo, opr2->as_register_lo());
2682
#else
2683
      // cpu register - cpu register
2684
      Register ylo = opr2->as_register_lo();
2685
      Register yhi = opr2->as_register_hi();
2686
      __ subl(xlo, ylo);
2687
      __ sbbl(xhi, yhi);
2688
      if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2689
        __ orl(xhi, xlo);
2690
      }
2691
#endif // _LP64
2692
    } else if (opr2->is_constant()) {
2693
      // cpu register - constant 0
2694
      assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2695
#ifdef _LP64
2696
      __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2697
#else
2698
      assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case");
2699
      __ orl(xhi, xlo);
2700
#endif // _LP64
2701
    } else {
2702
      ShouldNotReachHere();
2703
    }
2704

2705
  } else if (opr1->is_single_xmm()) {
2706
    XMMRegister reg1 = opr1->as_xmm_float_reg();
2707
    if (opr2->is_single_xmm()) {
2708
      // xmm register - xmm register
2709
      __ ucomiss(reg1, opr2->as_xmm_float_reg());
2710
    } else if (opr2->is_stack()) {
2711
      // xmm register - stack
2712
      __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2713
    } else if (opr2->is_constant()) {
2714
      // xmm register - constant
2715
      __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2716
    } else if (opr2->is_address()) {
2717
      // xmm register - address
2718
      if (op->info() != nullptr) {
2719
        add_debug_info_for_null_check_here(op->info());
2720
      }
2721
      __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2722
    } else {
2723
      ShouldNotReachHere();
2724
    }
2725

2726
  } else if (opr1->is_double_xmm()) {
2727
    XMMRegister reg1 = opr1->as_xmm_double_reg();
2728
    if (opr2->is_double_xmm()) {
2729
      // xmm register - xmm register
2730
      __ ucomisd(reg1, opr2->as_xmm_double_reg());
2731
    } else if (opr2->is_stack()) {
2732
      // xmm register - stack
2733
      __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2734
    } else if (opr2->is_constant()) {
2735
      // xmm register - constant
2736
      __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2737
    } else if (opr2->is_address()) {
2738
      // xmm register - address
2739
      if (op->info() != nullptr) {
2740
        add_debug_info_for_null_check_here(op->info());
2741
      }
2742
      __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2743
    } else {
2744
      ShouldNotReachHere();
2745
    }
2746

2747
#ifndef _LP64
2748
  } else if(opr1->is_single_fpu() || opr1->is_double_fpu()) {
2749
    assert(opr1->is_fpu_register() && opr1->fpu() == 0, "currently left-hand side must be on TOS (relax this restriction)");
2750
    assert(opr2->is_fpu_register(), "both must be registers");
2751
    __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2752
#endif // LP64
2753

2754
  } else if (opr1->is_address() && opr2->is_constant()) {
2755
    LIR_Const* c = opr2->as_constant_ptr();
2756
#ifdef _LP64
2757
    if (is_reference_type(c->type())) {
2758
      assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2759
      __ movoop(rscratch1, c->as_jobject());
2760
    }
2761
#endif // LP64
2762
    if (op->info() != nullptr) {
2763
      add_debug_info_for_null_check_here(op->info());
2764
    }
2765
    // special case: address - constant
2766
    LIR_Address* addr = opr1->as_address_ptr();
2767
    if (c->type() == T_INT) {
2768
      __ cmpl(as_Address(addr), c->as_jint());
2769
    } else if (is_reference_type(c->type())) {
2770
#ifdef _LP64
2771
      // %%% Make this explode if addr isn't reachable until we figure out a
2772
      // better strategy by giving noreg as the temp for as_Address
2773
      __ cmpoop(rscratch1, as_Address(addr, noreg));
2774
#else
2775
      __ cmpoop(as_Address(addr), c->as_jobject());
2776
#endif // _LP64
2777
    } else {
2778
      ShouldNotReachHere();
2779
    }
2780

2781
  } else {
2782
    ShouldNotReachHere();
2783
  }
2784
}
2785

2786
void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2787
  if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2788
    if (left->is_single_xmm()) {
2789
      assert(right->is_single_xmm(), "must match");
2790
      __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2791
    } else if (left->is_double_xmm()) {
2792
      assert(right->is_double_xmm(), "must match");
2793
      __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2794

2795
    } else {
2796
#ifdef _LP64
2797
      ShouldNotReachHere();
2798
#else
2799
      assert(left->is_single_fpu() || left->is_double_fpu(), "must be");
2800
      assert(right->is_single_fpu() || right->is_double_fpu(), "must match");
2801

2802
      assert(left->fpu() == 0, "left must be on TOS");
2803
      __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(),
2804
                  op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2805
#endif // LP64
2806
    }
2807
  } else {
2808
    assert(code == lir_cmp_l2i, "check");
2809
#ifdef _LP64
2810
    Label done;
2811
    Register dest = dst->as_register();
2812
    __ cmpptr(left->as_register_lo(), right->as_register_lo());
2813
    __ movl(dest, -1);
2814
    __ jccb(Assembler::less, done);
2815
    __ setb(Assembler::notZero, dest);
2816
    __ movzbl(dest, dest);
2817
    __ bind(done);
2818
#else
2819
    __ lcmp2int(left->as_register_hi(),
2820
                left->as_register_lo(),
2821
                right->as_register_hi(),
2822
                right->as_register_lo());
2823
    move_regs(left->as_register_hi(), dst->as_register());
2824
#endif // _LP64
2825
  }
2826
}
2827

2828

2829
void LIR_Assembler::align_call(LIR_Code code) {
2830
  // make sure that the displacement word of the call ends up word aligned
2831
  int offset = __ offset();
2832
  switch (code) {
2833
  case lir_static_call:
2834
  case lir_optvirtual_call:
2835
  case lir_dynamic_call:
2836
    offset += NativeCall::displacement_offset;
2837
    break;
2838
  case lir_icvirtual_call:
2839
    offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2840
    break;
2841
  default: ShouldNotReachHere();
2842
  }
2843
  __ align(BytesPerWord, offset);
2844
}
2845

2846

2847
void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2848
  assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2849
         "must be aligned");
2850
  __ call(AddressLiteral(op->addr(), rtype));
2851
  add_call_info(code_offset(), op->info());
2852
  __ post_call_nop();
2853
}
2854

2855

2856
void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2857
  __ ic_call(op->addr());
2858
  add_call_info(code_offset(), op->info());
2859
  assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2860
         "must be aligned");
2861
  __ post_call_nop();
2862
}
2863

2864

2865
void LIR_Assembler::emit_static_call_stub() {
2866
  address call_pc = __ pc();
2867
  address stub = __ start_a_stub(call_stub_size());
2868
  if (stub == nullptr) {
2869
    bailout("static call stub overflow");
2870
    return;
2871
  }
2872

2873
  int start = __ offset();
2874

2875
  // make sure that the displacement word of the call ends up word aligned
2876
  __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2877
  __ relocate(static_stub_Relocation::spec(call_pc));
2878
  __ mov_metadata(rbx, (Metadata*)nullptr);
2879
  // must be set to -1 at code generation time
2880
  assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2881
  // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2882
  __ jump(RuntimeAddress(__ pc()));
2883

2884
  assert(__ offset() - start <= call_stub_size(), "stub too big");
2885
  __ end_a_stub();
2886
}
2887

2888

2889
void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2890
  assert(exceptionOop->as_register() == rax, "must match");
2891
  assert(exceptionPC->as_register() == rdx, "must match");
2892

2893
  // exception object is not added to oop map by LinearScan
2894
  // (LinearScan assumes that no oops are in fixed registers)
2895
  info->add_register_oop(exceptionOop);
2896
  Runtime1::StubID unwind_id;
2897

2898
  // get current pc information
2899
  // pc is only needed if the method has an exception handler, the unwind code does not need it.
2900
  int pc_for_athrow_offset = __ offset();
2901
  InternalAddress pc_for_athrow(__ pc());
2902
  __ lea(exceptionPC->as_register(), pc_for_athrow);
2903
  add_call_info(pc_for_athrow_offset, info); // for exception handler
2904

2905
  __ verify_not_null_oop(rax);
2906
  // search an exception handler (rax: exception oop, rdx: throwing pc)
2907
  if (compilation()->has_fpu_code()) {
2908
    unwind_id = Runtime1::handle_exception_id;
2909
  } else {
2910
    unwind_id = Runtime1::handle_exception_nofpu_id;
2911
  }
2912
  __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2913

2914
  // enough room for two byte trap
2915
  __ nop();
2916
}
2917

2918

2919
void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2920
  assert(exceptionOop->as_register() == rax, "must match");
2921

2922
  __ jmp(_unwind_handler_entry);
2923
}
2924

2925

2926
void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2927

2928
  // optimized version for linear scan:
2929
  // * count must be already in ECX (guaranteed by LinearScan)
2930
  // * left and dest must be equal
2931
  // * tmp must be unused
2932
  assert(count->as_register() == SHIFT_count, "count must be in ECX");
2933
  assert(left == dest, "left and dest must be equal");
2934
  assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2935

2936
  if (left->is_single_cpu()) {
2937
    Register value = left->as_register();
2938
    assert(value != SHIFT_count, "left cannot be ECX");
2939

2940
    switch (code) {
2941
      case lir_shl:  __ shll(value); break;
2942
      case lir_shr:  __ sarl(value); break;
2943
      case lir_ushr: __ shrl(value); break;
2944
      default: ShouldNotReachHere();
2945
    }
2946
  } else if (left->is_double_cpu()) {
2947
    Register lo = left->as_register_lo();
2948
    Register hi = left->as_register_hi();
2949
    assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2950
#ifdef _LP64
2951
    switch (code) {
2952
      case lir_shl:  __ shlptr(lo);        break;
2953
      case lir_shr:  __ sarptr(lo);        break;
2954
      case lir_ushr: __ shrptr(lo);        break;
2955
      default: ShouldNotReachHere();
2956
    }
2957
#else
2958

2959
    switch (code) {
2960
      case lir_shl:  __ lshl(hi, lo);        break;
2961
      case lir_shr:  __ lshr(hi, lo, true);  break;
2962
      case lir_ushr: __ lshr(hi, lo, false); break;
2963
      default: ShouldNotReachHere();
2964
    }
2965
#endif // LP64
2966
  } else {
2967
    ShouldNotReachHere();
2968
  }
2969
}
2970

2971

2972
void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2973
  if (dest->is_single_cpu()) {
2974
    // first move left into dest so that left is not destroyed by the shift
2975
    Register value = dest->as_register();
2976
    count = count & 0x1F; // Java spec
2977

2978
    move_regs(left->as_register(), value);
2979
    switch (code) {
2980
      case lir_shl:  __ shll(value, count); break;
2981
      case lir_shr:  __ sarl(value, count); break;
2982
      case lir_ushr: __ shrl(value, count); break;
2983
      default: ShouldNotReachHere();
2984
    }
2985
  } else if (dest->is_double_cpu()) {
2986
#ifndef _LP64
2987
    Unimplemented();
2988
#else
2989
    // first move left into dest so that left is not destroyed by the shift
2990
    Register value = dest->as_register_lo();
2991
    count = count & 0x1F; // Java spec
2992

2993
    move_regs(left->as_register_lo(), value);
2994
    switch (code) {
2995
      case lir_shl:  __ shlptr(value, count); break;
2996
      case lir_shr:  __ sarptr(value, count); break;
2997
      case lir_ushr: __ shrptr(value, count); break;
2998
      default: ShouldNotReachHere();
2999
    }
3000
#endif // _LP64
3001
  } else {
3002
    ShouldNotReachHere();
3003
  }
3004
}
3005

3006

3007
void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
3008
  assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3009
  int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3010
  assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3011
  __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
3012
}
3013

3014

3015
void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
3016
  assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3017
  int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3018
  assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3019
  __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3020
}
3021

3022

3023
void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
3024
  assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3025
  int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3026
  assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3027
  __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
3028
}
3029

3030

3031
void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
3032
  assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3033
  int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3034
  assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3035
  __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
3036
}
3037

3038

3039
// This code replaces a call to arraycopy; no exception may
3040
// be thrown in this code, they must be thrown in the System.arraycopy
3041
// activation frame; we could save some checks if this would not be the case
3042
void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3043
  ciArrayKlass* default_type = op->expected_type();
3044
  Register src = op->src()->as_register();
3045
  Register dst = op->dst()->as_register();
3046
  Register src_pos = op->src_pos()->as_register();
3047
  Register dst_pos = op->dst_pos()->as_register();
3048
  Register length  = op->length()->as_register();
3049
  Register tmp = op->tmp()->as_register();
3050
  Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3051

3052
  CodeStub* stub = op->stub();
3053
  int flags = op->flags();
3054
  BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
3055
  if (is_reference_type(basic_type)) basic_type = T_OBJECT;
3056

3057
  // if we don't know anything, just go through the generic arraycopy
3058
  if (default_type == nullptr) {
3059
    // save outgoing arguments on stack in case call to System.arraycopy is needed
3060
    // HACK ALERT. This code used to push the parameters in a hardwired fashion
3061
    // for interpreter calling conventions. Now we have to do it in new style conventions.
3062
    // For the moment until C1 gets the new register allocator I just force all the
3063
    // args to the right place (except the register args) and then on the back side
3064
    // reload the register args properly if we go slow path. Yuck
3065

3066
    // These are proper for the calling convention
3067
    store_parameter(length, 2);
3068
    store_parameter(dst_pos, 1);
3069
    store_parameter(dst, 0);
3070

3071
    // these are just temporary placements until we need to reload
3072
    store_parameter(src_pos, 3);
3073
    store_parameter(src, 4);
3074
    NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3075

3076
    address copyfunc_addr = StubRoutines::generic_arraycopy();
3077
    assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
3078

3079
    // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
3080
#ifdef _LP64
3081
    // The arguments are in java calling convention so we can trivially shift them to C
3082
    // convention
3083
    assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
3084
    __ mov(c_rarg0, j_rarg0);
3085
    assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
3086
    __ mov(c_rarg1, j_rarg1);
3087
    assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
3088
    __ mov(c_rarg2, j_rarg2);
3089
    assert_different_registers(c_rarg3, j_rarg4);
3090
    __ mov(c_rarg3, j_rarg3);
3091
#ifdef _WIN64
3092
    // Allocate abi space for args but be sure to keep stack aligned
3093
    __ subptr(rsp, 6*wordSize);
3094
    store_parameter(j_rarg4, 4);
3095
#ifndef PRODUCT
3096
    if (PrintC1Statistics) {
3097
      __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
3098
    }
3099
#endif
3100
    __ call(RuntimeAddress(copyfunc_addr));
3101
    __ addptr(rsp, 6*wordSize);
3102
#else
3103
    __ mov(c_rarg4, j_rarg4);
3104
#ifndef PRODUCT
3105
    if (PrintC1Statistics) {
3106
      __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
3107
    }
3108
#endif
3109
    __ call(RuntimeAddress(copyfunc_addr));
3110
#endif // _WIN64
3111
#else
3112
    __ push(length);
3113
    __ push(dst_pos);
3114
    __ push(dst);
3115
    __ push(src_pos);
3116
    __ push(src);
3117

3118
#ifndef PRODUCT
3119
    if (PrintC1Statistics) {
3120
      __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
3121
    }
3122
#endif
3123
    __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack
3124

3125
#endif // _LP64
3126

3127
    __ testl(rax, rax);
3128
    __ jcc(Assembler::equal, *stub->continuation());
3129

3130
    __ mov(tmp, rax);
3131
    __ xorl(tmp, -1);
3132

3133
    // Reload values from the stack so they are where the stub
3134
    // expects them.
3135
    __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3136
    __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3137
    __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3138
    __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3139
    __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3140

3141
    __ subl(length, tmp);
3142
    __ addl(src_pos, tmp);
3143
    __ addl(dst_pos, tmp);
3144
    __ jmp(*stub->entry());
3145

3146
    __ bind(*stub->continuation());
3147
    return;
3148
  }
3149

3150
  assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3151

3152
  int elem_size = type2aelembytes(basic_type);
3153
  Address::ScaleFactor scale;
3154

3155
  switch (elem_size) {
3156
    case 1 :
3157
      scale = Address::times_1;
3158
      break;
3159
    case 2 :
3160
      scale = Address::times_2;
3161
      break;
3162
    case 4 :
3163
      scale = Address::times_4;
3164
      break;
3165
    case 8 :
3166
      scale = Address::times_8;
3167
      break;
3168
    default:
3169
      scale = Address::no_scale;
3170
      ShouldNotReachHere();
3171
  }
3172

3173
  Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
3174
  Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
3175
  Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
3176
  Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3177

3178
  // length and pos's are all sign extended at this point on 64bit
3179

3180
  // test for null
3181
  if (flags & LIR_OpArrayCopy::src_null_check) {
3182
    __ testptr(src, src);
3183
    __ jcc(Assembler::zero, *stub->entry());
3184
  }
3185
  if (flags & LIR_OpArrayCopy::dst_null_check) {
3186
    __ testptr(dst, dst);
3187
    __ jcc(Assembler::zero, *stub->entry());
3188
  }
3189

3190
  // If the compiler was not able to prove that exact type of the source or the destination
3191
  // of the arraycopy is an array type, check at runtime if the source or the destination is
3192
  // an instance type.
3193
  if (flags & LIR_OpArrayCopy::type_check) {
3194
    if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3195
      __ load_klass(tmp, dst, tmp_load_klass);
3196
      __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3197
      __ jcc(Assembler::greaterEqual, *stub->entry());
3198
    }
3199

3200
    if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3201
      __ load_klass(tmp, src, tmp_load_klass);
3202
      __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3203
      __ jcc(Assembler::greaterEqual, *stub->entry());
3204
    }
3205
  }
3206

3207
  // check if negative
3208
  if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
3209
    __ testl(src_pos, src_pos);
3210
    __ jcc(Assembler::less, *stub->entry());
3211
  }
3212
  if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
3213
    __ testl(dst_pos, dst_pos);
3214
    __ jcc(Assembler::less, *stub->entry());
3215
  }
3216

3217
  if (flags & LIR_OpArrayCopy::src_range_check) {
3218
    __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
3219
    __ cmpl(tmp, src_length_addr);
3220
    __ jcc(Assembler::above, *stub->entry());
3221
  }
3222
  if (flags & LIR_OpArrayCopy::dst_range_check) {
3223
    __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3224
    __ cmpl(tmp, dst_length_addr);
3225
    __ jcc(Assembler::above, *stub->entry());
3226
  }
3227

3228
  if (flags & LIR_OpArrayCopy::length_positive_check) {
3229
    __ testl(length, length);
3230
    __ jcc(Assembler::less, *stub->entry());
3231
  }
3232

3233
#ifdef _LP64
3234
  __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3235
  __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3236
#endif
3237

3238
  if (flags & LIR_OpArrayCopy::type_check) {
3239
    // We don't know the array types are compatible
3240
    if (basic_type != T_OBJECT) {
3241
      // Simple test for basic type arrays
3242
      if (UseCompressedClassPointers) {
3243
        __ movl(tmp, src_klass_addr);
3244
        __ cmpl(tmp, dst_klass_addr);
3245
      } else {
3246
        __ movptr(tmp, src_klass_addr);
3247
        __ cmpptr(tmp, dst_klass_addr);
3248
      }
3249
      __ jcc(Assembler::notEqual, *stub->entry());
3250
    } else {
3251
      // For object arrays, if src is a sub class of dst then we can
3252
      // safely do the copy.
3253
      Label cont, slow;
3254

3255
      __ push(src);
3256
      __ push(dst);
3257

3258
      __ load_klass(src, src, tmp_load_klass);
3259
      __ load_klass(dst, dst, tmp_load_klass);
3260

3261
      __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
3262

3263
      __ push(src);
3264
      __ push(dst);
3265
      __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
3266
      __ pop(dst);
3267
      __ pop(src);
3268

3269
      __ testl(src, src);
3270
      __ jcc(Assembler::notEqual, cont);
3271

3272
      __ bind(slow);
3273
      __ pop(dst);
3274
      __ pop(src);
3275

3276
      address copyfunc_addr = StubRoutines::checkcast_arraycopy();
3277
      if (copyfunc_addr != nullptr) { // use stub if available
3278
        // src is not a sub class of dst so we have to do a
3279
        // per-element check.
3280

3281
        int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
3282
        if ((flags & mask) != mask) {
3283
          // Check that at least both of them object arrays.
3284
          assert(flags & mask, "one of the two should be known to be an object array");
3285

3286
          if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3287
            __ load_klass(tmp, src, tmp_load_klass);
3288
          } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3289
            __ load_klass(tmp, dst, tmp_load_klass);
3290
          }
3291
          int lh_offset = in_bytes(Klass::layout_helper_offset());
3292
          Address klass_lh_addr(tmp, lh_offset);
3293
          jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
3294
          __ cmpl(klass_lh_addr, objArray_lh);
3295
          __ jcc(Assembler::notEqual, *stub->entry());
3296
        }
3297

3298
       // Spill because stubs can use any register they like and it's
3299
       // easier to restore just those that we care about.
3300
       store_parameter(dst, 0);
3301
       store_parameter(dst_pos, 1);
3302
       store_parameter(length, 2);
3303
       store_parameter(src_pos, 3);
3304
       store_parameter(src, 4);
3305

3306
#ifndef _LP64
3307
        __ movptr(tmp, dst_klass_addr);
3308
        __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset()));
3309
        __ push(tmp);
3310
        __ movl(tmp, Address(tmp, Klass::super_check_offset_offset()));
3311
        __ push(tmp);
3312
        __ push(length);
3313
        __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3314
        __ push(tmp);
3315
        __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3316
        __ push(tmp);
3317

3318
        __ call_VM_leaf(copyfunc_addr, 5);
3319
#else
3320
        __ movl2ptr(length, length); //higher 32bits must be null
3321

3322
        __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3323
        assert_different_registers(c_rarg0, dst, dst_pos, length);
3324
        __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3325
        assert_different_registers(c_rarg1, dst, length);
3326

3327
        __ mov(c_rarg2, length);
3328
        assert_different_registers(c_rarg2, dst);
3329

3330
#ifdef _WIN64
3331
        // Allocate abi space for args but be sure to keep stack aligned
3332
        __ subptr(rsp, 6*wordSize);
3333
        __ load_klass(c_rarg3, dst, tmp_load_klass);
3334
        __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
3335
        store_parameter(c_rarg3, 4);
3336
        __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
3337
        __ call(RuntimeAddress(copyfunc_addr));
3338
        __ addptr(rsp, 6*wordSize);
3339
#else
3340
        __ load_klass(c_rarg4, dst, tmp_load_klass);
3341
        __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
3342
        __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
3343
        __ call(RuntimeAddress(copyfunc_addr));
3344
#endif
3345

3346
#endif
3347

3348
#ifndef PRODUCT
3349
        if (PrintC1Statistics) {
3350
          Label failed;
3351
          __ testl(rax, rax);
3352
          __ jcc(Assembler::notZero, failed);
3353
          __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
3354
          __ bind(failed);
3355
        }
3356
#endif
3357

3358
        __ testl(rax, rax);
3359
        __ jcc(Assembler::zero, *stub->continuation());
3360

3361
#ifndef PRODUCT
3362
        if (PrintC1Statistics) {
3363
          __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
3364
        }
3365
#endif
3366

3367
        __ mov(tmp, rax);
3368

3369
        __ xorl(tmp, -1);
3370

3371
        // Restore previously spilled arguments
3372
        __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3373
        __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3374
        __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3375
        __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3376
        __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3377

3378

3379
        __ subl(length, tmp);
3380
        __ addl(src_pos, tmp);
3381
        __ addl(dst_pos, tmp);
3382
      }
3383

3384
      __ jmp(*stub->entry());
3385

3386
      __ bind(cont);
3387
      __ pop(dst);
3388
      __ pop(src);
3389
    }
3390
  }
3391

3392
#ifdef ASSERT
3393
  if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3394
    // Sanity check the known type with the incoming class.  For the
3395
    // primitive case the types must match exactly with src.klass and
3396
    // dst.klass each exactly matching the default type.  For the
3397
    // object array case, if no type check is needed then either the
3398
    // dst type is exactly the expected type and the src type is a
3399
    // subtype which we can't check or src is the same array as dst
3400
    // but not necessarily exactly of type default_type.
3401
    Label known_ok, halt;
3402
    __ mov_metadata(tmp, default_type->constant_encoding());
3403
#ifdef _LP64
3404
    if (UseCompressedClassPointers) {
3405
      __ encode_klass_not_null(tmp, rscratch1);
3406
    }
3407
#endif
3408

3409
    if (basic_type != T_OBJECT) {
3410

3411
      if (UseCompressedClassPointers)          __ cmpl(tmp, dst_klass_addr);
3412
      else                   __ cmpptr(tmp, dst_klass_addr);
3413
      __ jcc(Assembler::notEqual, halt);
3414
      if (UseCompressedClassPointers)          __ cmpl(tmp, src_klass_addr);
3415
      else                   __ cmpptr(tmp, src_klass_addr);
3416
      __ jcc(Assembler::equal, known_ok);
3417
    } else {
3418
      if (UseCompressedClassPointers)          __ cmpl(tmp, dst_klass_addr);
3419
      else                   __ cmpptr(tmp, dst_klass_addr);
3420
      __ jcc(Assembler::equal, known_ok);
3421
      __ cmpptr(src, dst);
3422
      __ jcc(Assembler::equal, known_ok);
3423
    }
3424
    __ bind(halt);
3425
    __ stop("incorrect type information in arraycopy");
3426
    __ bind(known_ok);
3427
  }
3428
#endif
3429

3430
#ifndef PRODUCT
3431
  if (PrintC1Statistics) {
3432
    __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
3433
  }
3434
#endif
3435

3436
#ifdef _LP64
3437
  assert_different_registers(c_rarg0, dst, dst_pos, length);
3438
  __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3439
  assert_different_registers(c_rarg1, length);
3440
  __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3441
  __ mov(c_rarg2, length);
3442

3443
#else
3444
  __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3445
  store_parameter(tmp, 0);
3446
  __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3447
  store_parameter(tmp, 1);
3448
  store_parameter(length, 2);
3449
#endif // _LP64
3450

3451
  bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
3452
  bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
3453
  const char *name;
3454
  address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
3455
  __ call_VM_leaf(entry, 0);
3456

3457
  if (stub != nullptr) {
3458
    __ bind(*stub->continuation());
3459
  }
3460
}
3461

3462
void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3463
  assert(op->crc()->is_single_cpu(),  "crc must be register");
3464
  assert(op->val()->is_single_cpu(),  "byte value must be register");
3465
  assert(op->result_opr()->is_single_cpu(), "result must be register");
3466
  Register crc = op->crc()->as_register();
3467
  Register val = op->val()->as_register();
3468
  Register res = op->result_opr()->as_register();
3469

3470
  assert_different_registers(val, crc, res);
3471

3472
  __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
3473
  __ notl(crc); // ~crc
3474
  __ update_byte_crc32(crc, val, res);
3475
  __ notl(crc); // ~crc
3476
  __ mov(res, crc);
3477
}
3478

3479
void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3480
  Register obj = op->obj_opr()->as_register();  // may not be an oop
3481
  Register hdr = op->hdr_opr()->as_register();
3482
  Register lock = op->lock_opr()->as_register();
3483
  if (LockingMode == LM_MONITOR) {
3484
    if (op->info() != nullptr) {
3485
      add_debug_info_for_null_check_here(op->info());
3486
      __ null_check(obj);
3487
    }
3488
    __ jmp(*op->stub()->entry());
3489
  } else if (op->code() == lir_lock) {
3490
    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3491
    Register tmp = LockingMode == LM_LIGHTWEIGHT ? op->scratch_opr()->as_register() : noreg;
3492
    // add debug info for NullPointerException only if one is possible
3493
    int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
3494
    if (op->info() != nullptr) {
3495
      add_debug_info_for_null_check(null_check_offset, op->info());
3496
    }
3497
    // done
3498
  } else if (op->code() == lir_unlock) {
3499
    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3500
    __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3501
  } else {
3502
    Unimplemented();
3503
  }
3504
  __ bind(*op->stub()->continuation());
3505
}
3506

3507
void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
3508
  Register obj = op->obj()->as_pointer_register();
3509
  Register result = op->result_opr()->as_pointer_register();
3510

3511
  CodeEmitInfo* info = op->info();
3512
  if (info != nullptr) {
3513
    add_debug_info_for_null_check_here(info);
3514
  }
3515

3516
#ifdef _LP64
3517
  if (UseCompressedClassPointers) {
3518
    __ movl(result, Address(obj, oopDesc::klass_offset_in_bytes()));
3519
    __ decode_klass_not_null(result, rscratch1);
3520
  } else
3521
#endif
3522
    __ movptr(result, Address(obj, oopDesc::klass_offset_in_bytes()));
3523
}
3524

3525
void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3526
  ciMethod* method = op->profiled_method();
3527
  int bci          = op->profiled_bci();
3528
  ciMethod* callee = op->profiled_callee();
3529
  Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3530

3531
  // Update counter for all call types
3532
  ciMethodData* md = method->method_data_or_null();
3533
  assert(md != nullptr, "Sanity");
3534
  ciProfileData* data = md->bci_to_data(bci);
3535
  assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
3536
  assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
3537
  Register mdo  = op->mdo()->as_register();
3538
  __ mov_metadata(mdo, md->constant_encoding());
3539
  Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3540
  // Perform additional virtual call profiling for invokevirtual and
3541
  // invokeinterface bytecodes
3542
  if (op->should_profile_receiver_type()) {
3543
    assert(op->recv()->is_single_cpu(), "recv must be allocated");
3544
    Register recv = op->recv()->as_register();
3545
    assert_different_registers(mdo, recv);
3546
    assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3547
    ciKlass* known_klass = op->known_holder();
3548
    if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
3549
      // We know the type that will be seen at this call site; we can
3550
      // statically update the MethodData* rather than needing to do
3551
      // dynamic tests on the receiver type
3552

3553
      // NOTE: we should probably put a lock around this search to
3554
      // avoid collisions by concurrent compilations
3555
      ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3556
      uint i;
3557
      for (i = 0; i < VirtualCallData::row_limit(); i++) {
3558
        ciKlass* receiver = vc_data->receiver(i);
3559
        if (known_klass->equals(receiver)) {
3560
          Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3561
          __ addptr(data_addr, DataLayout::counter_increment);
3562
          return;
3563
        }
3564
      }
3565

3566
      // Receiver type not found in profile data; select an empty slot
3567

3568
      // Note that this is less efficient than it should be because it
3569
      // always does a write to the receiver part of the
3570
      // VirtualCallData rather than just the first time
3571
      for (i = 0; i < VirtualCallData::row_limit(); i++) {
3572
        ciKlass* receiver = vc_data->receiver(i);
3573
        if (receiver == nullptr) {
3574
          Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3575
          __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
3576
          Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3577
          __ addptr(data_addr, DataLayout::counter_increment);
3578
          return;
3579
        }
3580
      }
3581
    } else {
3582
      __ load_klass(recv, recv, tmp_load_klass);
3583
      Label update_done;
3584
      type_profile_helper(mdo, md, data, recv, &update_done);
3585
      // Receiver did not match any saved receiver and there is no empty row for it.
3586
      // Increment total counter to indicate polymorphic case.
3587
      __ addptr(counter_addr, DataLayout::counter_increment);
3588

3589
      __ bind(update_done);
3590
    }
3591
  } else {
3592
    // Static call
3593
    __ addptr(counter_addr, DataLayout::counter_increment);
3594
  }
3595
}
3596

3597
void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3598
  Register obj = op->obj()->as_register();
3599
  Register tmp = op->tmp()->as_pointer_register();
3600
  Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3601
  Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3602
  ciKlass* exact_klass = op->exact_klass();
3603
  intptr_t current_klass = op->current_klass();
3604
  bool not_null = op->not_null();
3605
  bool no_conflict = op->no_conflict();
3606

3607
  Label update, next, none;
3608

3609
  bool do_null = !not_null;
3610
  bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3611
  bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3612

3613
  assert(do_null || do_update, "why are we here?");
3614
  assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3615

3616
  __ verify_oop(obj);
3617

3618
#ifdef ASSERT
3619
  if (obj == tmp) {
3620
#ifdef _LP64
3621
    assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3622
#else
3623
    assert_different_registers(obj, mdo_addr.base(), mdo_addr.index());
3624
#endif
3625
  } else {
3626
#ifdef _LP64
3627
    assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3628
#else
3629
    assert_different_registers(obj, tmp, mdo_addr.base(), mdo_addr.index());
3630
#endif
3631
  }
3632
#endif
3633
  if (do_null) {
3634
    __ testptr(obj, obj);
3635
    __ jccb(Assembler::notZero, update);
3636
    if (!TypeEntries::was_null_seen(current_klass)) {
3637
      __ testptr(mdo_addr, TypeEntries::null_seen);
3638
#ifndef ASSERT
3639
      __ jccb(Assembler::notZero, next); // already set
3640
#else
3641
      __ jcc(Assembler::notZero, next); // already set
3642
#endif
3643
      // atomic update to prevent overwriting Klass* with 0
3644
      __ lock();
3645
      __ orptr(mdo_addr, TypeEntries::null_seen);
3646
    }
3647
    if (do_update) {
3648
#ifndef ASSERT
3649
      __ jmpb(next);
3650
    }
3651
#else
3652
      __ jmp(next);
3653
    }
3654
  } else {
3655
    __ testptr(obj, obj);
3656
    __ jcc(Assembler::notZero, update);
3657
    __ stop("unexpected null obj");
3658
#endif
3659
  }
3660

3661
  __ bind(update);
3662

3663
  if (do_update) {
3664
#ifdef ASSERT
3665
    if (exact_klass != nullptr) {
3666
      Label ok;
3667
      __ load_klass(tmp, obj, tmp_load_klass);
3668
      __ push(tmp);
3669
      __ mov_metadata(tmp, exact_klass->constant_encoding());
3670
      __ cmpptr(tmp, Address(rsp, 0));
3671
      __ jcc(Assembler::equal, ok);
3672
      __ stop("exact klass and actual klass differ");
3673
      __ bind(ok);
3674
      __ pop(tmp);
3675
    }
3676
#endif
3677
    if (!no_conflict) {
3678
      if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3679
        if (exact_klass != nullptr) {
3680
          __ mov_metadata(tmp, exact_klass->constant_encoding());
3681
        } else {
3682
          __ load_klass(tmp, obj, tmp_load_klass);
3683
        }
3684
#ifdef _LP64
3685
        __ mov(rscratch1, tmp); // save original value before XOR
3686
#endif
3687
        __ xorptr(tmp, mdo_addr);
3688
        __ testptr(tmp, TypeEntries::type_klass_mask);
3689
        // klass seen before, nothing to do. The unknown bit may have been
3690
        // set already but no need to check.
3691
        __ jccb(Assembler::zero, next);
3692

3693
        __ testptr(tmp, TypeEntries::type_unknown);
3694
        __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3695

3696
        if (TypeEntries::is_type_none(current_klass)) {
3697
          __ testptr(mdo_addr, TypeEntries::type_mask);
3698
          __ jccb(Assembler::zero, none);
3699
#ifdef _LP64
3700
          // There is a chance that the checks above (re-reading profiling
3701
          // data from memory) fail if another thread has just set the
3702
          // profiling to this obj's klass
3703
          __ mov(tmp, rscratch1); // get back original value before XOR
3704
          __ xorptr(tmp, mdo_addr);
3705
          __ testptr(tmp, TypeEntries::type_klass_mask);
3706
          __ jccb(Assembler::zero, next);
3707
#endif
3708
        }
3709
      } else {
3710
        assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3711
               ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3712

3713
        __ testptr(mdo_addr, TypeEntries::type_unknown);
3714
        __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3715
      }
3716

3717
      // different than before. Cannot keep accurate profile.
3718
      __ orptr(mdo_addr, TypeEntries::type_unknown);
3719

3720
      if (TypeEntries::is_type_none(current_klass)) {
3721
        __ jmpb(next);
3722

3723
        __ bind(none);
3724
        // first time here. Set profile type.
3725
        __ movptr(mdo_addr, tmp);
3726
#ifdef ASSERT
3727
        __ andptr(tmp, TypeEntries::type_klass_mask);
3728
        __ verify_klass_ptr(tmp);
3729
#endif
3730
      }
3731
    } else {
3732
      // There's a single possible klass at this profile point
3733
      assert(exact_klass != nullptr, "should be");
3734
      if (TypeEntries::is_type_none(current_klass)) {
3735
        __ mov_metadata(tmp, exact_klass->constant_encoding());
3736
        __ xorptr(tmp, mdo_addr);
3737
        __ testptr(tmp, TypeEntries::type_klass_mask);
3738
#ifdef ASSERT
3739
        __ jcc(Assembler::zero, next);
3740

3741
        {
3742
          Label ok;
3743
          __ push(tmp);
3744
          __ testptr(mdo_addr, TypeEntries::type_mask);
3745
          __ jcc(Assembler::zero, ok);
3746
          // may have been set by another thread
3747
          __ mov_metadata(tmp, exact_klass->constant_encoding());
3748
          __ xorptr(tmp, mdo_addr);
3749
          __ testptr(tmp, TypeEntries::type_mask);
3750
          __ jcc(Assembler::zero, ok);
3751

3752
          __ stop("unexpected profiling mismatch");
3753
          __ bind(ok);
3754
          __ pop(tmp);
3755
        }
3756
#else
3757
        __ jccb(Assembler::zero, next);
3758
#endif
3759
        // first time here. Set profile type.
3760
        __ movptr(mdo_addr, tmp);
3761
#ifdef ASSERT
3762
        __ andptr(tmp, TypeEntries::type_klass_mask);
3763
        __ verify_klass_ptr(tmp);
3764
#endif
3765
      } else {
3766
        assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3767
               ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3768

3769
        __ testptr(mdo_addr, TypeEntries::type_unknown);
3770
        __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3771

3772
        __ orptr(mdo_addr, TypeEntries::type_unknown);
3773
      }
3774
    }
3775
  }
3776
  __ bind(next);
3777
}
3778

3779
void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3780
  Unimplemented();
3781
}
3782

3783

3784
void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3785
  __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3786
}
3787

3788

3789
void LIR_Assembler::align_backward_branch_target() {
3790
  __ align(BytesPerWord);
3791
}
3792

3793

3794
void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3795
  if (left->is_single_cpu()) {
3796
    __ negl(left->as_register());
3797
    move_regs(left->as_register(), dest->as_register());
3798

3799
  } else if (left->is_double_cpu()) {
3800
    Register lo = left->as_register_lo();
3801
#ifdef _LP64
3802
    Register dst = dest->as_register_lo();
3803
    __ movptr(dst, lo);
3804
    __ negptr(dst);
3805
#else
3806
    Register hi = left->as_register_hi();
3807
    __ lneg(hi, lo);
3808
    if (dest->as_register_lo() == hi) {
3809
      assert(dest->as_register_hi() != lo, "destroying register");
3810
      move_regs(hi, dest->as_register_hi());
3811
      move_regs(lo, dest->as_register_lo());
3812
    } else {
3813
      move_regs(lo, dest->as_register_lo());
3814
      move_regs(hi, dest->as_register_hi());
3815
    }
3816
#endif // _LP64
3817

3818
  } else if (dest->is_single_xmm()) {
3819
#ifdef _LP64
3820
    if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3821
      assert(tmp->is_valid(), "need temporary");
3822
      assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg());
3823
      __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2);
3824
    }
3825
    else
3826
#endif
3827
    {
3828
      assert(!tmp->is_valid(), "do not need temporary");
3829
      if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3830
        __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3831
      }
3832
      __ xorps(dest->as_xmm_float_reg(),
3833
               ExternalAddress((address)float_signflip_pool),
3834
               rscratch1);
3835
    }
3836
  } else if (dest->is_double_xmm()) {
3837
#ifdef _LP64
3838
    if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3839
      assert(tmp->is_valid(), "need temporary");
3840
      assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg());
3841
      __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2);
3842
    }
3843
    else
3844
#endif
3845
    {
3846
      assert(!tmp->is_valid(), "do not need temporary");
3847
      if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3848
        __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3849
      }
3850
      __ xorpd(dest->as_xmm_double_reg(),
3851
               ExternalAddress((address)double_signflip_pool),
3852
               rscratch1);
3853
    }
3854
#ifndef _LP64
3855
  } else if (left->is_single_fpu() || left->is_double_fpu()) {
3856
    assert(left->fpu() == 0, "arg must be on TOS");
3857
    assert(dest->fpu() == 0, "dest must be TOS");
3858
    __ fchs();
3859
#endif // !_LP64
3860

3861
  } else {
3862
    ShouldNotReachHere();
3863
  }
3864
}
3865

3866

3867
void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3868
  assert(src->is_address(), "must be an address");
3869
  assert(dest->is_register(), "must be a register");
3870

3871
  PatchingStub* patch = nullptr;
3872
  if (patch_code != lir_patch_none) {
3873
    patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3874
  }
3875

3876
  Register reg = dest->as_pointer_register();
3877
  LIR_Address* addr = src->as_address_ptr();
3878
  __ lea(reg, as_Address(addr));
3879

3880
  if (patch != nullptr) {
3881
    patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3882
  }
3883
}
3884

3885

3886

3887
void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3888
  assert(!tmp->is_valid(), "don't need temporary");
3889
  __ call(RuntimeAddress(dest));
3890
  if (info != nullptr) {
3891
    add_call_info_here(info);
3892
  }
3893
  __ post_call_nop();
3894
}
3895

3896

3897
void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3898
  assert(type == T_LONG, "only for volatile long fields");
3899

3900
  if (info != nullptr) {
3901
    add_debug_info_for_null_check_here(info);
3902
  }
3903

3904
  if (src->is_double_xmm()) {
3905
    if (dest->is_double_cpu()) {
3906
#ifdef _LP64
3907
      __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3908
#else
3909
      __ movdl(dest->as_register_lo(), src->as_xmm_double_reg());
3910
      __ psrlq(src->as_xmm_double_reg(), 32);
3911
      __ movdl(dest->as_register_hi(), src->as_xmm_double_reg());
3912
#endif // _LP64
3913
    } else if (dest->is_double_stack()) {
3914
      __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3915
    } else if (dest->is_address()) {
3916
      __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3917
    } else {
3918
      ShouldNotReachHere();
3919
    }
3920

3921
  } else if (dest->is_double_xmm()) {
3922
    if (src->is_double_stack()) {
3923
      __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3924
    } else if (src->is_address()) {
3925
      __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3926
    } else {
3927
      ShouldNotReachHere();
3928
    }
3929

3930
#ifndef _LP64
3931
  } else if (src->is_double_fpu()) {
3932
    assert(src->fpu_regnrLo() == 0, "must be TOS");
3933
    if (dest->is_double_stack()) {
3934
      __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix()));
3935
    } else if (dest->is_address()) {
3936
      __ fistp_d(as_Address(dest->as_address_ptr()));
3937
    } else {
3938
      ShouldNotReachHere();
3939
    }
3940

3941
  } else if (dest->is_double_fpu()) {
3942
    assert(dest->fpu_regnrLo() == 0, "must be TOS");
3943
    if (src->is_double_stack()) {
3944
      __ fild_d(frame_map()->address_for_slot(src->double_stack_ix()));
3945
    } else if (src->is_address()) {
3946
      __ fild_d(as_Address(src->as_address_ptr()));
3947
    } else {
3948
      ShouldNotReachHere();
3949
    }
3950
#endif // !_LP64
3951

3952
  } else {
3953
    ShouldNotReachHere();
3954
  }
3955
}
3956

3957
#ifdef ASSERT
3958
// emit run-time assertion
3959
void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3960
  assert(op->code() == lir_assert, "must be");
3961

3962
  if (op->in_opr1()->is_valid()) {
3963
    assert(op->in_opr2()->is_valid(), "both operands must be valid");
3964
    comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3965
  } else {
3966
    assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3967
    assert(op->condition() == lir_cond_always, "no other conditions allowed");
3968
  }
3969

3970
  Label ok;
3971
  if (op->condition() != lir_cond_always) {
3972
    Assembler::Condition acond = Assembler::zero;
3973
    switch (op->condition()) {
3974
      case lir_cond_equal:        acond = Assembler::equal;       break;
3975
      case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
3976
      case lir_cond_less:         acond = Assembler::less;        break;
3977
      case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
3978
      case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3979
      case lir_cond_greater:      acond = Assembler::greater;     break;
3980
      case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
3981
      case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
3982
      default:                    ShouldNotReachHere();
3983
    }
3984
    __ jcc(acond, ok);
3985
  }
3986
  if (op->halt()) {
3987
    const char* str = __ code_string(op->msg());
3988
    __ stop(str);
3989
  } else {
3990
    breakpoint();
3991
  }
3992
  __ bind(ok);
3993
}
3994
#endif
3995

3996
void LIR_Assembler::membar() {
3997
  // QQQ sparc TSO uses this,
3998
  __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3999
}
4000

4001
void LIR_Assembler::membar_acquire() {
4002
  // No x86 machines currently require load fences
4003
}
4004

4005
void LIR_Assembler::membar_release() {
4006
  // No x86 machines currently require store fences
4007
}
4008

4009
void LIR_Assembler::membar_loadload() {
4010
  // no-op
4011
  //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
4012
}
4013

4014
void LIR_Assembler::membar_storestore() {
4015
  // no-op
4016
  //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
4017
}
4018

4019
void LIR_Assembler::membar_loadstore() {
4020
  // no-op
4021
  //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
4022
}
4023

4024
void LIR_Assembler::membar_storeload() {
4025
  __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
4026
}
4027

4028
void LIR_Assembler::on_spin_wait() {
4029
  __ pause ();
4030
}
4031

4032
void LIR_Assembler::get_thread(LIR_Opr result_reg) {
4033
  assert(result_reg->is_register(), "check");
4034
#ifdef _LP64
4035
  // __ get_thread(result_reg->as_register_lo());
4036
  __ mov(result_reg->as_register(), r15_thread);
4037
#else
4038
  __ get_thread(result_reg->as_register());
4039
#endif // _LP64
4040
}
4041

4042

4043
void LIR_Assembler::peephole(LIR_List*) {
4044
  // do nothing for now
4045
}
4046

4047
void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
4048
  assert(data == dest, "xchg/xadd uses only 2 operands");
4049

4050
  if (data->type() == T_INT) {
4051
    if (code == lir_xadd) {
4052
      __ lock();
4053
      __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
4054
    } else {
4055
      __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
4056
    }
4057
  } else if (data->is_oop()) {
4058
    assert (code == lir_xchg, "xadd for oops");
4059
    Register obj = data->as_register();
4060
#ifdef _LP64
4061
    if (UseCompressedOops) {
4062
      __ encode_heap_oop(obj);
4063
      __ xchgl(obj, as_Address(src->as_address_ptr()));
4064
      __ decode_heap_oop(obj);
4065
    } else {
4066
      __ xchgptr(obj, as_Address(src->as_address_ptr()));
4067
    }
4068
#else
4069
    __ xchgl(obj, as_Address(src->as_address_ptr()));
4070
#endif
4071
  } else if (data->type() == T_LONG) {
4072
#ifdef _LP64
4073
    assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
4074
    if (code == lir_xadd) {
4075
      __ lock();
4076
      __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
4077
    } else {
4078
      __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
4079
    }
4080
#else
4081
    ShouldNotReachHere();
4082
#endif
4083
  } else {
4084
    ShouldNotReachHere();
4085
  }
4086
}
4087

4088
#undef __
4089