jdk

1
/*
2
 * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4
 *
5
 * This code is free software; you can redistribute it and/or modify it
6
 * under the terms of the GNU General Public License version 2 only, as
7
 * published by the Free Software Foundation.
8
 *
9
 * This code is distributed in the hope that it will be useful, but WITHOUT
10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12
 * version 2 for more details (a copy is included in the LICENSE file that
13
 * accompanied this code).
14
 *
15
 * You should have received a copy of the GNU General Public License version
16
 * 2 along with this work; if not, write to the Free Software Foundation,
17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18
 *
19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20
 * or visit www.oracle.com if you need additional information or have any
21
 * questions.
22
 */
23

24
#include "precompiled.hpp"
25
#include "asm/macroAssembler.hpp"
26
#include "code/codeBlob.hpp"
27
#include "code/codeBlob.hpp"
28
#include "code/vmreg.inline.hpp"
29
#include "compiler/disassembler.hpp"
30
#include "logging/logStream.hpp"
31
#include "memory/resourceArea.hpp"
32
#include "prims/foreignGlobals.inline.hpp"
33
#include "prims/upcallLinker.hpp"
34
#include "runtime/sharedRuntime.hpp"
35
#include "runtime/signature.hpp"
36
#include "runtime/stubRoutines.hpp"
37
#include "utilities/formatBuffer.hpp"
38
#include "utilities/globalDefinitions.hpp"
39

40
#define __ _masm->
41

42
static bool is_valid_XMM(XMMRegister reg) {
43
  return reg->is_valid() && (reg->encoding() < (UseAVX >= 3 ? 32 : 16)); // why is this not covered by is_valid()?
44
}
45

46
static bool is_valid_gp(Register reg) {
47
  return reg->is_valid() && (reg->encoding() < (UseAPX ? 32 : 16));
48
}
49

50
// for callee saved regs, according to the caller's ABI
51
static int compute_reg_save_area_size(const ABIDescriptor& abi) {
52
  int size = 0;
53
  for (Register reg = as_Register(0); is_valid_gp(reg); reg = reg->successor()) {
54
    if (reg == rbp || reg == rsp) continue; // saved/restored by prologue/epilogue
55
    if (!abi.is_volatile_reg(reg)) {
56
      size += 8; // bytes
57
    }
58
  }
59

60
  for (XMMRegister reg = as_XMMRegister(0); is_valid_XMM(reg); reg = reg->successor()) {
61
    if (!abi.is_volatile_reg(reg)) {
62
      if (UseAVX >= 3) {
63
        size += 64; // bytes
64
      } else if (UseAVX >= 1) {
65
        size += 32;
66
      } else {
67
        size += 16;
68
      }
69
    }
70
  }
71

72
#ifndef _WIN64
73
  // for mxcsr
74
  size += 8;
75
#endif
76

77
  return size;
78
}
79

80
constexpr int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
81

82
static void preserve_callee_saved_registers(MacroAssembler* _masm, const ABIDescriptor& abi, int reg_save_area_offset) {
83
  // 1. iterate all registers in the architecture
84
  //     - check if they are volatile or not for the given abi
85
  //     - if NOT, we need to save it here
86
  // 2. save mxcsr on non-windows platforms
87

88
  int offset = reg_save_area_offset;
89

90
  __ block_comment("{ preserve_callee_saved_regs ");
91
  for (Register reg = as_Register(0); is_valid_gp(reg); reg = reg->successor()) {
92
    if (reg == rbp || reg == rsp) continue; // saved/restored by prologue/epilogue
93
    if (!abi.is_volatile_reg(reg)) {
94
      __ movptr(Address(rsp, offset), reg);
95
      offset += 8;
96
    }
97
  }
98

99
  for (XMMRegister reg = as_XMMRegister(0); is_valid_XMM(reg); reg = reg->successor()) {
100
    if (!abi.is_volatile_reg(reg)) {
101
      if (UseAVX >= 3) {
102
        __ evmovdqul(Address(rsp, offset), reg, Assembler::AVX_512bit);
103
        offset += 64;
104
      } else if (UseAVX >= 1) {
105
        __ vmovdqu(Address(rsp, offset), reg);
106
        offset += 32;
107
      } else {
108
        __ movdqu(Address(rsp, offset), reg);
109
        offset += 16;
110
      }
111
    }
112
  }
113

114
#ifndef _WIN64
115
  {
116
    const Address mxcsr_save(rsp, offset);
117
    Label skip_ldmx;
118
    __ stmxcsr(mxcsr_save);
119
    __ movl(rax, mxcsr_save);
120
    __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
121
    ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
122
    __ cmp32(rax, mxcsr_std, rscratch1);
123
    __ jcc(Assembler::equal, skip_ldmx);
124
    __ ldmxcsr(mxcsr_std, rscratch1);
125
    __ bind(skip_ldmx);
126
  }
127
#endif
128

129
  __ block_comment("} preserve_callee_saved_regs ");
130
}
131

132
static void restore_callee_saved_registers(MacroAssembler* _masm, const ABIDescriptor& abi, int reg_save_area_offset) {
133
  // 1. iterate all registers in the architecture
134
  //     - check if they are volatile or not for the given abi
135
  //     - if NOT, we need to restore it here
136
  // 2. restore mxcsr on non-windows platforms
137

138
  int offset = reg_save_area_offset;
139

140
  __ block_comment("{ restore_callee_saved_regs ");
141
  for (Register reg = as_Register(0); is_valid_gp(reg); reg = reg->successor()) {
142
    if (reg == rbp || reg == rsp) continue; // saved/restored by prologue/epilogue
143
    if (!abi.is_volatile_reg(reg)) {
144
      __ movptr(reg, Address(rsp, offset));
145
      offset += 8;
146
    }
147
  }
148

149
  for (XMMRegister reg = as_XMMRegister(0); is_valid_XMM(reg); reg = reg->successor()) {
150
    if (!abi.is_volatile_reg(reg)) {
151
      if (UseAVX >= 3) {
152
        __ evmovdqul(reg, Address(rsp, offset), Assembler::AVX_512bit);
153
        offset += 64;
154
      } else if (UseAVX >= 1) {
155
        __ vmovdqu(reg, Address(rsp, offset));
156
        offset += 32;
157
      } else {
158
        __ movdqu(reg, Address(rsp, offset));
159
        offset += 16;
160
      }
161
    }
162
  }
163

164
#ifndef _WIN64
165
  const Address mxcsr_save(rsp, offset);
166
  __ ldmxcsr(mxcsr_save);
167
#endif
168

169
  __ block_comment("} restore_callee_saved_regs ");
170
}
171

172
static const int upcall_stub_code_base_size = 1024;
173
static const int upcall_stub_size_per_arg = 16;
174

175
address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
176
                                       BasicType* out_sig_bt, int total_out_args,
177
                                       BasicType ret_type,
178
                                       jobject jabi, jobject jconv,
179
                                       bool needs_return_buffer, int ret_buf_size) {
180
  const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi);
181
  const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv);
182
  int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg);
183
  CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 1);
184
  if (buffer.blob() == nullptr) {
185
    return nullptr;
186
  }
187

188
  GrowableArray<VMStorage> unfiltered_out_regs;
189
  int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs);
190
  int preserved_bytes = SharedRuntime::out_preserve_stack_slots() * VMRegImpl::stack_slot_size;
191
  int stack_bytes = preserved_bytes + out_arg_bytes;
192
  int out_arg_area = align_up(stack_bytes , StackAlignmentInBytes);
193

194
  // out_arg_area (for stack arguments) doubles as shadow space for native calls.
195
  // make sure it is big enough.
196
  if (out_arg_area < frame::arg_reg_save_area_bytes) {
197
    out_arg_area = frame::arg_reg_save_area_bytes;
198
  }
199

200
  int reg_save_area_size = compute_reg_save_area_size(abi);
201
  RegSpiller arg_spiller(call_regs._arg_regs);
202
  RegSpiller result_spiller(call_regs._ret_regs);
203

204
  int shuffle_area_offset    = 0;
205
  int res_save_area_offset   = shuffle_area_offset    + out_arg_area;
206
  int arg_save_area_offset   = res_save_area_offset   + result_spiller.spill_size_bytes();
207
  int reg_save_area_offset   = arg_save_area_offset   + arg_spiller.spill_size_bytes();
208
  int frame_data_offset      = reg_save_area_offset   + reg_save_area_size;
209
  int frame_bottom_offset    = frame_data_offset      + sizeof(UpcallStub::FrameData);
210

211
  StubLocations locs;
212
  int ret_buf_offset = -1;
213
  if (needs_return_buffer) {
214
    ret_buf_offset = frame_bottom_offset;
215
    frame_bottom_offset += ret_buf_size;
216
    // use a free register for shuffling code to pick up return
217
    // buffer address from
218
    locs.set(StubLocations::RETURN_BUFFER, abi._scratch1);
219
  }
220

221
  VMStorage shuffle_reg = as_VMStorage(rbx);
222
  GrowableArray<VMStorage> in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs);
223
  GrowableArray<VMStorage> filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs);
224
  ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, shuffle_reg);
225

226
#ifndef PRODUCT
227
  LogTarget(Trace, foreign, upcall) lt;
228
  if (lt.is_enabled()) {
229
    ResourceMark rm;
230
    LogStream ls(lt);
231
    arg_shuffle.print_on(&ls);
232
  }
233
#endif
234

235
  int frame_size = frame_bottom_offset;
236
  frame_size = align_up(frame_size, StackAlignmentInBytes);
237

238
  // Ok The space we have allocated will look like:
239
  //
240
  //
241
  // FP-> |                     |
242
  //      |---------------------| = frame_bottom_offset = frame_size
243
  //      | (optional)          |
244
  //      | ret_buf             |
245
  //      |---------------------| = ret_buf_offset
246
  //      |                     |
247
  //      | FrameData           |
248
  //      |---------------------| = frame_data_offset
249
  //      |                     |
250
  //      | reg_save_area       |
251
  //      |---------------------| = reg_save_are_offset
252
  //      |                     |
253
  //      | arg_save_area       |
254
  //      |---------------------| = arg_save_are_offset
255
  //      |                     |
256
  //      | res_save_area       |
257
  //      |---------------------| = res_save_are_offset
258
  //      |                     |
259
  // SP-> | out_arg_area        |   needs to be at end for shadow space
260
  //
261
  //
262

263
  //////////////////////////////////////////////////////////////////////////////
264

265
  MacroAssembler* _masm = new MacroAssembler(&buffer);
266
  address start = __ pc();
267
  __ enter(); // set up frame
268
  if ((abi._stack_alignment_bytes % 16) != 0) {
269
    // stack alignment of caller is not a multiple of 16
270
    __ andptr(rsp, -StackAlignmentInBytes); // align stack
271
  }
272
  // allocate frame (frame_size is also aligned, so stack is still aligned)
273
  __ subptr(rsp, frame_size);
274

275
  // we have to always spill args since we need to do a call to get the thread
276
  // (and maybe attach it).
277
  arg_spiller.generate_spill(_masm, arg_save_area_offset);
278

279
  preserve_callee_saved_registers(_masm, abi, reg_save_area_offset);
280

281
  __ block_comment("{ on_entry");
282
  __ vzeroupper();
283
  __ lea(c_rarg0, Address(rsp, frame_data_offset));
284
  __ movptr(c_rarg1, (intptr_t)receiver);
285
  // stack already aligned
286
  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::on_entry)));
287
  __ movptr(r15_thread, rax);
288
  __ reinit_heapbase();
289
  __ block_comment("} on_entry");
290

291
  __ block_comment("{ argument shuffle");
292
  arg_spiller.generate_fill(_masm, arg_save_area_offset);
293
  if (needs_return_buffer) {
294
    assert(ret_buf_offset != -1, "no return buffer allocated");
295
    __ lea(as_Register(locs.get(StubLocations::RETURN_BUFFER)), Address(rsp, ret_buf_offset));
296
  }
297
  arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, 0);
298
  __ block_comment("} argument shuffle");
299

300
  __ block_comment("{ receiver ");
301
  __ get_vm_result(j_rarg0, r15_thread);
302
  __ block_comment("} receiver ");
303

304
  __ mov_metadata(rbx, entry);
305
  __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx); // just in case callee is deoptimized
306

307
  __ push_cont_fastpath();
308

309
  __ call(Address(rbx, Method::from_compiled_offset()));
310

311
  __ pop_cont_fastpath();
312

313
  // return value shuffle
314
  if (!needs_return_buffer) {
315
#ifdef ASSERT
316
    if (call_regs._ret_regs.length() == 1) { // 0 or 1
317
      VMStorage j_expected_result_reg;
318
      switch (ret_type) {
319
        case T_BOOLEAN:
320
        case T_BYTE:
321
        case T_SHORT:
322
        case T_CHAR:
323
        case T_INT:
324
        case T_LONG:
325
        j_expected_result_reg = as_VMStorage(rax);
326
        break;
327
        case T_FLOAT:
328
        case T_DOUBLE:
329
          j_expected_result_reg = as_VMStorage(xmm0);
330
          break;
331
        default:
332
          fatal("unexpected return type: %s", type2name(ret_type));
333
      }
334
      // No need to move for now, since CallArranger can pick a return type
335
      // that goes in the same reg for both CCs. But, at least assert they are the same
336
      assert(call_regs._ret_regs.at(0) == j_expected_result_reg, "unexpected result register");
337
    }
338
#endif
339
  } else {
340
    assert(ret_buf_offset != -1, "no return buffer allocated");
341
    __ lea(rscratch1, Address(rsp, ret_buf_offset));
342
    int offset = 0;
343
    for (int i = 0; i < call_regs._ret_regs.length(); i++) {
344
      VMStorage reg = call_regs._ret_regs.at(i);
345
      if (reg.type() == StorageType::INTEGER) {
346
        __ movptr(as_Register(reg), Address(rscratch1, offset));
347
        offset += 8;
348
      } else if (reg.type() == StorageType::VECTOR) {
349
        __ movdqu(as_XMMRegister(reg), Address(rscratch1, offset));
350
        offset += 16;
351
      } else {
352
        ShouldNotReachHere();
353
      }
354
    }
355
  }
356

357
  result_spiller.generate_spill(_masm, res_save_area_offset);
358

359
  __ block_comment("{ on_exit");
360
  __ vzeroupper();
361
  __ lea(c_rarg0, Address(rsp, frame_data_offset));
362
  // stack already aligned
363
  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::on_exit)));
364
  __ reinit_heapbase();
365
  __ block_comment("} on_exit");
366

367
  restore_callee_saved_registers(_masm, abi, reg_save_area_offset);
368

369
  result_spiller.generate_fill(_masm, res_save_area_offset);
370

371
  __ leave();
372
  __ ret(0);
373

374
  //////////////////////////////////////////////////////////////////////////////
375

376
  _masm->flush();
377

378
#ifndef PRODUCT
379
  stringStream ss;
380
  ss.print("upcall_stub_%s", entry->signature()->as_C_string());
381
  const char* name = _masm->code_string(ss.freeze());
382
#else // PRODUCT
383
  const char* name = "upcall_stub";
384
#endif // PRODUCT
385

386
  buffer.log_section_sizes(name);
387

388
  UpcallStub* blob
389
    = UpcallStub::create(name,
390
                         &buffer,
391
                         receiver,
392
                         in_ByteSize(frame_data_offset));
393
  if (blob == nullptr) {
394
    return nullptr;
395
  }
396

397
#ifndef PRODUCT
398
  if (lt.is_enabled()) {
399
    ResourceMark rm;
400
    LogStream ls(lt);
401
    blob->print_on(&ls);
402
  }
403
#endif
404

405
  return blob->code_begin();
406
}
407