jdk

1
/*
2
 * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
3
 * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
4
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5
 *
6
 * This code is free software; you can redistribute it and/or modify it
7
 * under the terms of the GNU General Public License version 2 only, as
8
 * published by the Free Software Foundation.
9
 *
10
 * This code is distributed in the hope that it will be useful, but WITHOUT
11
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
13
 * version 2 for more details (a copy is included in the LICENSE file that
14
 * accompanied this code).
15
 *
16
 * You should have received a copy of the GNU General Public License version
17
 * 2 along with this work; if not, write to the Free Software Foundation,
18
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19
 *
20
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21
 * or visit www.oracle.com if you need additional information or have any
22
 * questions.
23
 *
24
 */
25

26
#include "precompiled.hpp"
27
#include "asm/assembler.hpp"
28
#include "c1/c1_CodeStubs.hpp"
29
#include "c1/c1_Defs.hpp"
30
#include "c1/c1_MacroAssembler.hpp"
31
#include "c1/c1_Runtime1.hpp"
32
#include "compiler/disassembler.hpp"
33
#include "compiler/oopMap.hpp"
34
#include "gc/shared/cardTable.hpp"
35
#include "gc/shared/cardTableBarrierSet.hpp"
36
#include "gc/shared/collectedHeap.hpp"
37
#include "gc/shared/tlab_globals.hpp"
38
#include "interpreter/interpreter.hpp"
39
#include "memory/universe.hpp"
40
#include "nativeInst_aarch64.hpp"
41
#include "oops/oop.inline.hpp"
42
#include "prims/jvmtiExport.hpp"
43
#include "register_aarch64.hpp"
44
#include "runtime/sharedRuntime.hpp"
45
#include "runtime/signature.hpp"
46
#include "runtime/stubRoutines.hpp"
47
#include "runtime/vframe.hpp"
48
#include "runtime/vframeArray.hpp"
49
#include "utilities/powerOfTwo.hpp"
50
#include "vmreg_aarch64.inline.hpp"
51

52

53
// Implementation of StubAssembler
54

55
int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
56
  // setup registers
57
  assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
58
  assert(oop_result1 != rthread && metadata_result != rthread, "registers must be different");
59
  assert(args_size >= 0, "illegal args_size");
60
  bool align_stack = false;
61

62
  mov(c_rarg0, rthread);
63
  set_num_rt_args(0); // Nothing on stack
64

65
  Label retaddr;
66
  set_last_Java_frame(sp, rfp, retaddr, rscratch1);
67

68
  // do the call
69
  lea(rscratch1, RuntimeAddress(entry));
70
  blr(rscratch1);
71
  bind(retaddr);
72
  int call_offset = offset();
73
  // verify callee-saved register
74
#ifdef ASSERT
75
  push(r0, sp);
76
  { Label L;
77
    get_thread(r0);
78
    cmp(rthread, r0);
79
    br(Assembler::EQ, L);
80
    stop("StubAssembler::call_RT: rthread not callee saved?");
81
    bind(L);
82
  }
83
  pop(r0, sp);
84
#endif
85
  reset_last_Java_frame(true);
86

87
  // check for pending exceptions
88
  { Label L;
89
    // check for pending exceptions (java_thread is set upon return)
90
    ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
91
    cbz(rscratch1, L);
92
    // exception pending => remove activation and forward to exception handler
93
    // make sure that the vm_results are cleared
94
    if (oop_result1->is_valid()) {
95
      str(zr, Address(rthread, JavaThread::vm_result_offset()));
96
    }
97
    if (metadata_result->is_valid()) {
98
      str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
99
    }
100
    if (frame_size() == no_frame_size) {
101
      leave();
102
      far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
103
    } else if (_stub_id == Runtime1::forward_exception_id) {
104
      should_not_reach_here();
105
    } else {
106
      far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
107
    }
108
    bind(L);
109
  }
110
  // get oop results if there are any and reset the values in the thread
111
  if (oop_result1->is_valid()) {
112
    get_vm_result(oop_result1, rthread);
113
  }
114
  if (metadata_result->is_valid()) {
115
    get_vm_result_2(metadata_result, rthread);
116
  }
117
  return call_offset;
118
}
119

120

121
int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
122
  mov(c_rarg1, arg1);
123
  return call_RT(oop_result1, metadata_result, entry, 1);
124
}
125

126

127
int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
128
  if (c_rarg1 == arg2) {
129
    if (c_rarg2 == arg1) {
130
      mov(rscratch1, arg1);
131
      mov(arg1, arg2);
132
      mov(arg2, rscratch1);
133
    } else {
134
      mov(c_rarg2, arg2);
135
      mov(c_rarg1, arg1);
136
    }
137
  } else {
138
    mov(c_rarg1, arg1);
139
    mov(c_rarg2, arg2);
140
  }
141
  return call_RT(oop_result1, metadata_result, entry, 2);
142
}
143

144

145
int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
146
  // if there is any conflict use the stack
147
  if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
148
      arg2 == c_rarg1 || arg2 == c_rarg3 ||
149
      arg3 == c_rarg1 || arg3 == c_rarg2) {
150
    stp(arg3, arg2, Address(pre(sp, -2 * wordSize)));
151
    stp(arg1, zr, Address(pre(sp, -2 * wordSize)));
152
    ldp(c_rarg1, zr, Address(post(sp, 2 * wordSize)));
153
    ldp(c_rarg3, c_rarg2, Address(post(sp, 2 * wordSize)));
154
  } else {
155
    mov(c_rarg1, arg1);
156
    mov(c_rarg2, arg2);
157
    mov(c_rarg3, arg3);
158
  }
159
  return call_RT(oop_result1, metadata_result, entry, 3);
160
}
161

162
enum return_state_t {
163
  does_not_return, requires_return
164
};
165

166

167
// Implementation of StubFrame
168

169
class StubFrame: public StackObj {
170
 private:
171
  StubAssembler* _sasm;
172
  bool _return_state;
173

174
 public:
175
  StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state=requires_return);
176
  void load_argument(int offset_in_words, Register reg);
177

178
  ~StubFrame();
179
};;
180

181
void StubAssembler::prologue(const char* name, bool must_gc_arguments) {
182
  set_info(name, must_gc_arguments);
183
  enter();
184
}
185

186
void StubAssembler::epilogue() {
187
  leave();
188
  ret(lr);
189
}
190

191
#define __ _sasm->
192

193
StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state) {
194
  _sasm = sasm;
195
  _return_state = return_state;
196
  __ prologue(name, must_gc_arguments);
197
}
198

199
// load parameters that were stored with LIR_Assembler::store_parameter
200
// Note: offsets for store_parameter and load_argument must match
201
void StubFrame::load_argument(int offset_in_words, Register reg) {
202
  __ load_parameter(offset_in_words, reg);
203
}
204

205
StubFrame::~StubFrame() {
206
  if (_return_state == requires_return) {
207
    __ epilogue();
208
  } else {
209
    __ should_not_reach_here();
210
  }
211
}
212

213
#undef __
214

215

216
// Implementation of Runtime1
217

218
#define __ sasm->
219

220
const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
221

222
// Stack layout for saving/restoring  all the registers needed during a runtime
223
// call (this includes deoptimization)
224
// Note: note that users of this frame may well have arguments to some runtime
225
// while these values are on the stack. These positions neglect those arguments
226
// but the code in save_live_registers will take the argument count into
227
// account.
228
//
229

230
enum reg_save_layout {
231
  reg_save_frame_size = 32 /* float */ + 32 /* integer */
232
};
233

234
// Save off registers which might be killed by calls into the runtime.
235
// Tries to smart of about FP registers.  In particular we separate
236
// saving and describing the FPU registers for deoptimization since we
237
// have to save the FPU registers twice if we describe them.  The
238
// deopt blob is the only thing which needs to describe FPU registers.
239
// In all other cases it should be sufficient to simply save their
240
// current value.
241

242
static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];
243
static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];
244
static int reg_save_size_in_words;
245
static int frame_size_in_bytes = -1;
246

247
static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
248
  int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;
249
  sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);
250
  int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
251
  OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
252

253
  for (int i = 0; i < FrameMap::nof_cpu_regs; i++) {
254
    Register r = as_Register(i);
255
    if (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding()) {
256
      int sp_offset = cpu_reg_save_offsets[i];
257
      oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
258
                                r->as_VMReg());
259
    }
260
  }
261

262
  if (save_fpu_registers) {
263
    for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
264
      FloatRegister r = as_FloatRegister(i);
265
      {
266
        int sp_offset = fpu_reg_save_offsets[i];
267
        oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
268
                                  r->as_VMReg());
269
      }
270
    }
271
  }
272
  return oop_map;
273
}
274

275
static OopMap* save_live_registers(StubAssembler* sasm,
276
                                   bool save_fpu_registers = true) {
277
  __ block_comment("save_live_registers");
278

279
  __ push(RegSet::range(r0, r29), sp);         // integer registers except lr & sp
280

281
  if (save_fpu_registers) {
282
    for (int i = 31; i>= 0; i -= 4) {
283
      __ sub(sp, sp, 4 * wordSize); // no pre-increment for st1. Emulate it without modifying other registers
284
      __ st1(as_FloatRegister(i-3), as_FloatRegister(i-2), as_FloatRegister(i-1),
285
          as_FloatRegister(i), __ T1D, Address(sp));
286
    }
287
  } else {
288
    __ add(sp, sp, -32 * wordSize);
289
  }
290

291
  return generate_oop_map(sasm, save_fpu_registers);
292
}
293

294
static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
295
  if (restore_fpu_registers) {
296
    for (int i = 0; i < 32; i += 4)
297
      __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
298
          as_FloatRegister(i+3), __ T1D, Address(__ post(sp, 4 * wordSize)));
299
  } else {
300
    __ add(sp, sp, 32 * wordSize);
301
  }
302

303
  __ pop(RegSet::range(r0, r29), sp);
304
}
305

306
static void restore_live_registers_except_r0(StubAssembler* sasm, bool restore_fpu_registers = true)  {
307

308
  if (restore_fpu_registers) {
309
    for (int i = 0; i < 32; i += 4)
310
      __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
311
          as_FloatRegister(i+3), __ T1D, Address(__ post(sp, 4 * wordSize)));
312
  } else {
313
    __ add(sp, sp, 32 * wordSize);
314
  }
315

316
  __ ldp(zr, r1, Address(__ post(sp, 16)));
317
  __ pop(RegSet::range(r2, r29), sp);
318
}
319

320

321

322
void Runtime1::initialize_pd() {
323
  int i;
324
  int sp_offset = 0;
325

326
  // all float registers are saved explicitly
327
  assert(FrameMap::nof_fpu_regs == 32, "double registers not handled here");
328
  for (i = 0; i < FrameMap::nof_fpu_regs; i++) {
329
    fpu_reg_save_offsets[i] = sp_offset;
330
    sp_offset += 2;   // SP offsets are in halfwords
331
  }
332

333
  for (i = 0; i < FrameMap::nof_cpu_regs; i++) {
334
    Register r = as_Register(i);
335
    cpu_reg_save_offsets[i] = sp_offset;
336
    sp_offset += 2;   // SP offsets are in halfwords
337
  }
338
}
339

340

341
// target: the entry point of the method that creates and posts the exception oop
342
// has_argument: true if the exception needs arguments (passed in rscratch1 and rscratch2)
343

344
OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
345
  // make a frame and preserve the caller's caller-save registers
346
  OopMap* oop_map = save_live_registers(sasm);
347
  int call_offset;
348
  if (!has_argument) {
349
    call_offset = __ call_RT(noreg, noreg, target);
350
  } else {
351
    __ mov(c_rarg1, rscratch1);
352
    __ mov(c_rarg2, rscratch2);
353
    call_offset = __ call_RT(noreg, noreg, target);
354
  }
355
  OopMapSet* oop_maps = new OopMapSet();
356
  oop_maps->add_gc_map(call_offset, oop_map);
357
  return oop_maps;
358
}
359

360

361
OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
362
  __ block_comment("generate_handle_exception");
363

364
  // incoming parameters
365
  const Register exception_oop = r0;
366
  const Register exception_pc  = r3;
367
  // other registers used in this stub
368

369
  // Save registers, if required.
370
  OopMapSet* oop_maps = new OopMapSet();
371
  OopMap* oop_map = nullptr;
372
  switch (id) {
373
  case forward_exception_id:
374
    // We're handling an exception in the context of a compiled frame.
375
    // The registers have been saved in the standard places.  Perform
376
    // an exception lookup in the caller and dispatch to the handler
377
    // if found.  Otherwise unwind and dispatch to the callers
378
    // exception handler.
379
    oop_map = generate_oop_map(sasm, 1 /*thread*/);
380

381
    // load and clear pending exception oop into r0
382
    __ ldr(exception_oop, Address(rthread, Thread::pending_exception_offset()));
383
    __ str(zr, Address(rthread, Thread::pending_exception_offset()));
384

385
    // load issuing PC (the return address for this stub) into r3
386
    __ ldr(exception_pc, Address(rfp, 1*BytesPerWord));
387
    __ authenticate_return_address(exception_pc);
388

389
    // make sure that the vm_results are cleared (may be unnecessary)
390
    __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
391
    __ str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
392
    break;
393
  case handle_exception_nofpu_id:
394
  case handle_exception_id:
395
    // At this point all registers MAY be live.
396
    oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);
397
    break;
398
  case handle_exception_from_callee_id: {
399
    // At this point all registers except exception oop (r0) and
400
    // exception pc (lr) are dead.
401
    const int frame_size = 2 /*fp, return address*/;
402
    oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
403
    sasm->set_frame_size(frame_size);
404
    break;
405
  }
406
  default: ShouldNotReachHere();
407
  }
408

409
  // verify that only r0 and r3 are valid at this time
410
  __ invalidate_registers(false, true, true, false, true, true);
411
  // verify that r0 contains a valid exception
412
  __ verify_not_null_oop(exception_oop);
413

414
#ifdef ASSERT
415
  // check that fields in JavaThread for exception oop and issuing pc are
416
  // empty before writing to them
417
  Label oop_empty;
418
  __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
419
  __ cbz(rscratch1, oop_empty);
420
  __ stop("exception oop already set");
421
  __ bind(oop_empty);
422

423
  Label pc_empty;
424
  __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
425
  __ cbz(rscratch1, pc_empty);
426
  __ stop("exception pc already set");
427
  __ bind(pc_empty);
428
#endif
429

430
  // save exception oop and issuing pc into JavaThread
431
  // (exception handler will load it from here)
432
  __ str(exception_oop, Address(rthread, JavaThread::exception_oop_offset()));
433
  __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));
434

435
  // patch throwing pc into return address (has bci & oop map)
436
  __ protect_return_address(exception_pc);
437
  __ str(exception_pc, Address(rfp, 1*BytesPerWord));
438

439
  // compute the exception handler.
440
  // the exception oop and the throwing pc are read from the fields in JavaThread
441
  int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
442
  oop_maps->add_gc_map(call_offset, oop_map);
443

444
  // r0: handler address
445
  //      will be the deopt blob if nmethod was deoptimized while we looked up
446
  //      handler regardless of whether handler existed in the nmethod.
447

448
  // only r0 is valid at this time, all other registers have been destroyed by the runtime call
449
  __ invalidate_registers(false, true, true, true, true, true);
450

451
  // patch the return address, this stub will directly return to the exception handler
452
  __ protect_return_address(r0);
453
  __ str(r0, Address(rfp, 1*BytesPerWord));
454

455
  switch (id) {
456
  case forward_exception_id:
457
  case handle_exception_nofpu_id:
458
  case handle_exception_id:
459
    // Restore the registers that were saved at the beginning.
460
    restore_live_registers(sasm, id != handle_exception_nofpu_id);
461
    break;
462
  case handle_exception_from_callee_id:
463
    break;
464
  default:  ShouldNotReachHere();
465
  }
466

467
  return oop_maps;
468
}
469

470

471
void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
472
  // incoming parameters
473
  const Register exception_oop = r0;
474
  // callee-saved copy of exception_oop during runtime call
475
  const Register exception_oop_callee_saved = r19;
476
  // other registers used in this stub
477
  const Register exception_pc = r3;
478
  const Register handler_addr = r1;
479

480
  if (AbortVMOnException) {
481
    __ mov(rscratch1, exception_oop);
482
    __ enter();
483
    save_live_registers(sasm);
484
    __ call_VM_leaf(CAST_FROM_FN_PTR(address, check_abort_on_vm_exception), rscratch1);
485
    restore_live_registers(sasm);
486
    __ leave();
487
  }
488

489
  // verify that only r0, is valid at this time
490
  __ invalidate_registers(false, true, true, true, true, true);
491

492
#ifdef ASSERT
493
  // check that fields in JavaThread for exception oop and issuing pc are empty
494
  Label oop_empty;
495
  __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
496
  __ cbz(rscratch1, oop_empty);
497
  __ stop("exception oop must be empty");
498
  __ bind(oop_empty);
499

500
  Label pc_empty;
501
  __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
502
  __ cbz(rscratch1, pc_empty);
503
  __ stop("exception pc must be empty");
504
  __ bind(pc_empty);
505
#endif
506

507
  // Save our return address because
508
  // exception_handler_for_return_address will destroy it.  We also
509
  // save exception_oop
510
  __ mov(r3, lr);
511
  __ protect_return_address();
512
  __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize)));
513

514
  // search the exception handler address of the caller (using the return address)
515
  __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, r3);
516
  // r0: exception handler address of the caller
517

518
  // Only R0 is valid at this time; all other registers have been
519
  // destroyed by the call.
520
  __ invalidate_registers(false, true, true, true, false, true);
521

522
  // move result of call into correct register
523
  __ mov(handler_addr, r0);
524

525
  // get throwing pc (= return address).
526
  // lr has been destroyed by the call
527
  __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize)));
528
  __ authenticate_return_address();
529
  __ mov(r3, lr);
530

531
  __ verify_not_null_oop(exception_oop);
532

533
  // continue at exception handler (return address removed)
534
  // note: do *not* remove arguments when unwinding the
535
  //       activation since the caller assumes having
536
  //       all arguments on the stack when entering the
537
  //       runtime to determine the exception handler
538
  //       (GC happens at call site with arguments!)
539
  // r0: exception oop
540
  // r3: throwing pc
541
  // r1: exception handler
542
  __ br(handler_addr);
543
}
544

545

546

547
OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
548
  // use the maximum number of runtime-arguments here because it is difficult to
549
  // distinguish each RT-Call.
550
  // Note: This number affects also the RT-Call in generate_handle_exception because
551
  //       the oop-map is shared for all calls.
552
  DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
553
  assert(deopt_blob != nullptr, "deoptimization blob must have been created");
554

555
  OopMap* oop_map = save_live_registers(sasm);
556

557
  __ mov(c_rarg0, rthread);
558
  Label retaddr;
559
  __ set_last_Java_frame(sp, rfp, retaddr, rscratch1);
560
  // do the call
561
  __ lea(rscratch1, RuntimeAddress(target));
562
  __ blr(rscratch1);
563
  __ bind(retaddr);
564
  OopMapSet* oop_maps = new OopMapSet();
565
  oop_maps->add_gc_map(__ offset(), oop_map);
566
  // verify callee-saved register
567
#ifdef ASSERT
568
  { Label L;
569
    __ get_thread(rscratch1);
570
    __ cmp(rthread, rscratch1);
571
    __ br(Assembler::EQ, L);
572
    __ stop("StubAssembler::call_RT: rthread not callee saved?");
573
    __ bind(L);
574
  }
575
#endif
576

577
  __ reset_last_Java_frame(true);
578

579
#ifdef ASSERT
580
  // check that fields in JavaThread for exception oop and issuing pc are empty
581
  Label oop_empty;
582
  __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
583
  __ cbz(rscratch1, oop_empty);
584
  __ stop("exception oop must be empty");
585
  __ bind(oop_empty);
586

587
  Label pc_empty;
588
  __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
589
  __ cbz(rscratch1, pc_empty);
590
  __ stop("exception pc must be empty");
591
  __ bind(pc_empty);
592
#endif
593

594
  // Runtime will return true if the nmethod has been deoptimized, this is the
595
  // expected scenario and anything else is  an error. Note that we maintain a
596
  // check on the result purely as a defensive measure.
597
  Label no_deopt;
598
  __ cbz(r0, no_deopt);                                // Have we deoptimized?
599

600
  // Perform a re-execute. The proper return  address is already on the stack,
601
  // we just need  to restore registers, pop  all of our frame  but the return
602
  // address and jump to the deopt blob.
603
  restore_live_registers(sasm);
604
  __ leave();
605
  __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
606

607
  __ bind(no_deopt);
608
  __ stop("deopt not performed");
609

610
  return oop_maps;
611
}
612

613

614
OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
615

616
  const Register exception_oop = r0;
617
  const Register exception_pc  = r3;
618

619
  // for better readability
620
  const bool must_gc_arguments = true;
621
  const bool dont_gc_arguments = false;
622

623
  // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
624
  bool save_fpu_registers = true;
625

626
  // stub code & info for the different stubs
627
  OopMapSet* oop_maps = nullptr;
628
  OopMap* oop_map = nullptr;
629
  switch (id) {
630
    {
631
    case forward_exception_id:
632
      {
633
        oop_maps = generate_handle_exception(id, sasm);
634
        __ leave();
635
        __ ret(lr);
636
      }
637
      break;
638

639
    case throw_div0_exception_id:
640
      { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments, does_not_return);
641
        oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
642
      }
643
      break;
644

645
    case throw_null_pointer_exception_id:
646
      { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments, does_not_return);
647
        oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
648
      }
649
      break;
650

651
    case new_instance_id:
652
    case fast_new_instance_id:
653
    case fast_new_instance_init_check_id:
654
      {
655
        Register klass = r3; // Incoming
656
        Register obj   = r0; // Result
657

658
        if (id == new_instance_id) {
659
          __ set_info("new_instance", dont_gc_arguments);
660
        } else if (id == fast_new_instance_id) {
661
          __ set_info("fast new_instance", dont_gc_arguments);
662
        } else {
663
          assert(id == fast_new_instance_init_check_id, "bad StubID");
664
          __ set_info("fast new_instance init check", dont_gc_arguments);
665
        }
666

667
        __ enter();
668
        OopMap* map = save_live_registers(sasm);
669
        int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
670
        oop_maps = new OopMapSet();
671
        oop_maps->add_gc_map(call_offset, map);
672
        restore_live_registers_except_r0(sasm);
673
        __ verify_oop(obj);
674
        __ leave();
675
        __ ret(lr);
676

677
        // r0,: new instance
678
      }
679

680
      break;
681

682
    case counter_overflow_id:
683
      {
684
        Register bci = r0, method = r1;
685
        __ enter();
686
        OopMap* map = save_live_registers(sasm);
687
        // Retrieve bci
688
        __ ldrw(bci, Address(rfp, 2*BytesPerWord));
689
        // And a pointer to the Method*
690
        __ ldr(method, Address(rfp, 3*BytesPerWord));
691
        int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
692
        oop_maps = new OopMapSet();
693
        oop_maps->add_gc_map(call_offset, map);
694
        restore_live_registers(sasm);
695
        __ leave();
696
        __ ret(lr);
697
      }
698
      break;
699

700
    case new_type_array_id:
701
    case new_object_array_id:
702
      {
703
        Register length   = r19; // Incoming
704
        Register klass    = r3; // Incoming
705
        Register obj      = r0; // Result
706

707
        if (id == new_type_array_id) {
708
          __ set_info("new_type_array", dont_gc_arguments);
709
        } else {
710
          __ set_info("new_object_array", dont_gc_arguments);
711
        }
712

713
#ifdef ASSERT
714
        // assert object type is really an array of the proper kind
715
        {
716
          Label ok;
717
          Register t0 = obj;
718
          __ ldrw(t0, Address(klass, Klass::layout_helper_offset()));
719
          __ asrw(t0, t0, Klass::_lh_array_tag_shift);
720
          int tag = ((id == new_type_array_id)
721
                     ? Klass::_lh_array_tag_type_value
722
                     : Klass::_lh_array_tag_obj_value);
723
          __ mov(rscratch1, tag);
724
          __ cmpw(t0, rscratch1);
725
          __ br(Assembler::EQ, ok);
726
          __ stop("assert(is an array klass)");
727
          __ should_not_reach_here();
728
          __ bind(ok);
729
        }
730
#endif // ASSERT
731

732
        __ enter();
733
        OopMap* map = save_live_registers(sasm);
734
        int call_offset;
735
        if (id == new_type_array_id) {
736
          call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
737
        } else {
738
          call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
739
        }
740

741
        oop_maps = new OopMapSet();
742
        oop_maps->add_gc_map(call_offset, map);
743
        restore_live_registers_except_r0(sasm);
744

745
        __ verify_oop(obj);
746
        __ leave();
747
        __ ret(lr);
748

749
        // r0: new array
750
      }
751
      break;
752

753
    case new_multi_array_id:
754
      { StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
755
        // r0,: klass
756
        // r19,: rank
757
        // r2: address of 1st dimension
758
        OopMap* map = save_live_registers(sasm);
759
        __ mov(c_rarg1, r0);
760
        __ mov(c_rarg3, r2);
761
        __ mov(c_rarg2, r19);
762
        int call_offset = __ call_RT(r0, noreg, CAST_FROM_FN_PTR(address, new_multi_array), r1, r2, r3);
763

764
        oop_maps = new OopMapSet();
765
        oop_maps->add_gc_map(call_offset, map);
766
        restore_live_registers_except_r0(sasm);
767

768
        // r0,: new multi array
769
        __ verify_oop(r0);
770
      }
771
      break;
772

773
    case register_finalizer_id:
774
      {
775
        __ set_info("register_finalizer", dont_gc_arguments);
776

777
        // This is called via call_runtime so the arguments
778
        // will be place in C abi locations
779

780
        __ verify_oop(c_rarg0);
781

782
        // load the klass and check the has finalizer flag
783
        Label register_finalizer;
784
        Register t = r5;
785
        __ load_klass(t, r0);
786
        __ ldrw(t, Address(t, Klass::access_flags_offset()));
787
        __ tbnz(t, exact_log2(JVM_ACC_HAS_FINALIZER), register_finalizer);
788
        __ ret(lr);
789

790
        __ bind(register_finalizer);
791
        __ enter();
792
        OopMap* oop_map = save_live_registers(sasm);
793
        int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), r0);
794
        oop_maps = new OopMapSet();
795
        oop_maps->add_gc_map(call_offset, oop_map);
796

797
        // Now restore all the live registers
798
        restore_live_registers(sasm);
799

800
        __ leave();
801
        __ ret(lr);
802
      }
803
      break;
804

805
    case throw_class_cast_exception_id:
806
      { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments, does_not_return);
807
        oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
808
      }
809
      break;
810

811
    case throw_incompatible_class_change_error_id:
812
      { StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments, does_not_return);
813
        oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
814
      }
815
      break;
816

817
    case slow_subtype_check_id:
818
      {
819
        // Typical calling sequence:
820
        // __ push(klass_RInfo);  // object klass or other subclass
821
        // __ push(sup_k_RInfo);  // array element klass or other superclass
822
        // __ bl(slow_subtype_check);
823
        // Note that the subclass is pushed first, and is therefore deepest.
824
        enum layout {
825
          r0_off, r0_off_hi,
826
          r2_off, r2_off_hi,
827
          r4_off, r4_off_hi,
828
          r5_off, r5_off_hi,
829
          sup_k_off, sup_k_off_hi,
830
          klass_off, klass_off_hi,
831
          framesize,
832
          result_off = sup_k_off
833
        };
834

835
        __ set_info("slow_subtype_check", dont_gc_arguments);
836
        __ push(RegSet::of(r0, r2, r4, r5), sp);
837

838
        // This is called by pushing args and not with C abi
839
        // __ ldr(r4, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass
840
        // __ ldr(r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass
841

842
        __ ldp(r4, r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size));
843

844
        Label miss;
845
        __ check_klass_subtype_slow_path(r4, r0, r2, r5, nullptr, &miss);
846

847
        // fallthrough on success:
848
        __ mov(rscratch1, 1);
849
        __ str(rscratch1, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
850
        __ pop(RegSet::of(r0, r2, r4, r5), sp);
851
        __ ret(lr);
852

853
        __ bind(miss);
854
        __ str(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
855
        __ pop(RegSet::of(r0, r2, r4, r5), sp);
856
        __ ret(lr);
857
      }
858
      break;
859

860
    case monitorenter_nofpu_id:
861
      save_fpu_registers = false;
862
      // fall through
863
    case monitorenter_id:
864
      {
865
        StubFrame f(sasm, "monitorenter", dont_gc_arguments);
866
        OopMap* map = save_live_registers(sasm, save_fpu_registers);
867

868
        // Called with store_parameter and not C abi
869

870
        f.load_argument(1, r0); // r0,: object
871
        f.load_argument(0, r1); // r1,: lock address
872

873
        int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), r0, r1);
874

875
        oop_maps = new OopMapSet();
876
        oop_maps->add_gc_map(call_offset, map);
877
        restore_live_registers(sasm, save_fpu_registers);
878
      }
879
      break;
880

881
    case monitorexit_nofpu_id:
882
      save_fpu_registers = false;
883
      // fall through
884
    case monitorexit_id:
885
      {
886
        StubFrame f(sasm, "monitorexit", dont_gc_arguments);
887
        OopMap* map = save_live_registers(sasm, save_fpu_registers);
888

889
        // Called with store_parameter and not C abi
890

891
        f.load_argument(0, r0); // r0,: lock address
892

893
        // note: really a leaf routine but must setup last java sp
894
        //       => use call_RT for now (speed can be improved by
895
        //       doing last java sp setup manually)
896
        int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), r0);
897

898
        oop_maps = new OopMapSet();
899
        oop_maps->add_gc_map(call_offset, map);
900
        restore_live_registers(sasm, save_fpu_registers);
901
      }
902
      break;
903

904
    case deoptimize_id:
905
      {
906
        StubFrame f(sasm, "deoptimize", dont_gc_arguments, does_not_return);
907
        OopMap* oop_map = save_live_registers(sasm);
908
        f.load_argument(0, c_rarg1);
909
        int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), c_rarg1);
910

911
        oop_maps = new OopMapSet();
912
        oop_maps->add_gc_map(call_offset, oop_map);
913
        restore_live_registers(sasm);
914
        DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
915
        assert(deopt_blob != nullptr, "deoptimization blob must have been created");
916
        __ leave();
917
        __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
918
      }
919
      break;
920

921
    case throw_range_check_failed_id:
922
      { StubFrame f(sasm, "range_check_failed", dont_gc_arguments, does_not_return);
923
        oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
924
      }
925
      break;
926

927
    case unwind_exception_id:
928
      { __ set_info("unwind_exception", dont_gc_arguments);
929
        // note: no stubframe since we are about to leave the current
930
        //       activation and we are calling a leaf VM function only.
931
        generate_unwind_exception(sasm);
932
      }
933
      break;
934

935
    case access_field_patching_id:
936
      { StubFrame f(sasm, "access_field_patching", dont_gc_arguments, does_not_return);
937
        // we should set up register map
938
        oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
939
      }
940
      break;
941

942
    case load_klass_patching_id:
943
      { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments, does_not_return);
944
        // we should set up register map
945
        oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
946
      }
947
      break;
948

949
    case load_mirror_patching_id:
950
      { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments, does_not_return);
951
        // we should set up register map
952
        oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
953
      }
954
      break;
955

956
    case load_appendix_patching_id:
957
      { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments, does_not_return);
958
        // we should set up register map
959
        oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
960
      }
961
      break;
962

963
    case handle_exception_nofpu_id:
964
    case handle_exception_id:
965
      { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
966
        oop_maps = generate_handle_exception(id, sasm);
967
      }
968
      break;
969

970
    case handle_exception_from_callee_id:
971
      { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
972
        oop_maps = generate_handle_exception(id, sasm);
973
      }
974
      break;
975

976
    case throw_index_exception_id:
977
      { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments, does_not_return);
978
        oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
979
      }
980
      break;
981

982
    case throw_array_store_exception_id:
983
      { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments, does_not_return);
984
        // tos + 0: link
985
        //     + 1: return address
986
        oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
987
      }
988
      break;
989

990
    case predicate_failed_trap_id:
991
      {
992
        StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments, does_not_return);
993

994
        OopMap* map = save_live_registers(sasm);
995

996
        int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
997
        oop_maps = new OopMapSet();
998
        oop_maps->add_gc_map(call_offset, map);
999
        restore_live_registers(sasm);
1000
        __ leave();
1001
        DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1002
        assert(deopt_blob != nullptr, "deoptimization blob must have been created");
1003

1004
        __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1005
      }
1006
      break;
1007

1008
    case dtrace_object_alloc_id:
1009
      { // c_rarg0: object
1010
        StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments);
1011
        save_live_registers(sasm);
1012

1013
        __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), c_rarg0);
1014

1015
        restore_live_registers(sasm);
1016
      }
1017
      break;
1018

1019
    default:
1020
      { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments, does_not_return);
1021
        __ mov(r0, (int)id);
1022
        __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1023
      }
1024
      break;
1025
    }
1026
  }
1027
  return oop_maps;
1028
}
1029

1030
#undef __
1031

1032
const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }
1033