2
* Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
3
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5
* This code is free software; you can redistribute it and/or modify it
6
* under the terms of the GNU General Public License version 2 only, as
7
* published by the Free Software Foundation.
9
* This code is distributed in the hope that it will be useful, but WITHOUT
10
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12
* version 2 for more details (a copy is included in the LICENSE file that
13
* accompanied this code).
15
* You should have received a copy of the GNU General Public License version
16
* 2 along with this work; if not, write to the Free Software Foundation,
17
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20
* or visit www.oracle.com if you need additional information or have any
24
#include "precompiled.hpp"
25
#include "asm/macroAssembler.hpp"
26
#include "code/codeBlob.hpp"
27
#include "code/codeBlob.hpp"
28
#include "code/vmreg.inline.hpp"
29
#include "compiler/disassembler.hpp"
30
#include "logging/logStream.hpp"
31
#include "memory/resourceArea.hpp"
32
#include "prims/foreignGlobals.inline.hpp"
33
#include "prims/upcallLinker.hpp"
34
#include "runtime/sharedRuntime.hpp"
35
#include "runtime/signature.hpp"
36
#include "runtime/stubRoutines.hpp"
37
#include "utilities/formatBuffer.hpp"
38
#include "utilities/globalDefinitions.hpp"
42
static bool is_valid_XMM(XMMRegister reg) {
43
return reg->is_valid() && (reg->encoding() < (UseAVX >= 3 ? 32 : 16)); // why is this not covered by is_valid()?
46
static bool is_valid_gp(Register reg) {
47
return reg->is_valid() && (reg->encoding() < (UseAPX ? 32 : 16));
50
// for callee saved regs, according to the caller's ABI
51
static int compute_reg_save_area_size(const ABIDescriptor& abi) {
53
for (Register reg = as_Register(0); is_valid_gp(reg); reg = reg->successor()) {
54
if (reg == rbp || reg == rsp) continue; // saved/restored by prologue/epilogue
55
if (!abi.is_volatile_reg(reg)) {
60
for (XMMRegister reg = as_XMMRegister(0); is_valid_XMM(reg); reg = reg->successor()) {
61
if (!abi.is_volatile_reg(reg)) {
64
} else if (UseAVX >= 1) {
80
constexpr int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions
82
static void preserve_callee_saved_registers(MacroAssembler* _masm, const ABIDescriptor& abi, int reg_save_area_offset) {
83
// 1. iterate all registers in the architecture
84
// - check if they are volatile or not for the given abi
85
// - if NOT, we need to save it here
86
// 2. save mxcsr on non-windows platforms
88
int offset = reg_save_area_offset;
90
__ block_comment("{ preserve_callee_saved_regs ");
91
for (Register reg = as_Register(0); is_valid_gp(reg); reg = reg->successor()) {
92
if (reg == rbp || reg == rsp) continue; // saved/restored by prologue/epilogue
93
if (!abi.is_volatile_reg(reg)) {
94
__ movptr(Address(rsp, offset), reg);
99
for (XMMRegister reg = as_XMMRegister(0); is_valid_XMM(reg); reg = reg->successor()) {
100
if (!abi.is_volatile_reg(reg)) {
102
__ evmovdqul(Address(rsp, offset), reg, Assembler::AVX_512bit);
104
} else if (UseAVX >= 1) {
105
__ vmovdqu(Address(rsp, offset), reg);
108
__ movdqu(Address(rsp, offset), reg);
116
const Address mxcsr_save(rsp, offset);
118
__ stmxcsr(mxcsr_save);
119
__ movl(rax, mxcsr_save);
120
__ andl(rax, MXCSR_MASK); // Only check control and mask bits
121
ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
122
__ cmp32(rax, mxcsr_std, rscratch1);
123
__ jcc(Assembler::equal, skip_ldmx);
124
__ ldmxcsr(mxcsr_std, rscratch1);
129
__ block_comment("} preserve_callee_saved_regs ");
132
static void restore_callee_saved_registers(MacroAssembler* _masm, const ABIDescriptor& abi, int reg_save_area_offset) {
133
// 1. iterate all registers in the architecture
134
// - check if they are volatile or not for the given abi
135
// - if NOT, we need to restore it here
136
// 2. restore mxcsr on non-windows platforms
138
int offset = reg_save_area_offset;
140
__ block_comment("{ restore_callee_saved_regs ");
141
for (Register reg = as_Register(0); is_valid_gp(reg); reg = reg->successor()) {
142
if (reg == rbp || reg == rsp) continue; // saved/restored by prologue/epilogue
143
if (!abi.is_volatile_reg(reg)) {
144
__ movptr(reg, Address(rsp, offset));
149
for (XMMRegister reg = as_XMMRegister(0); is_valid_XMM(reg); reg = reg->successor()) {
150
if (!abi.is_volatile_reg(reg)) {
152
__ evmovdqul(reg, Address(rsp, offset), Assembler::AVX_512bit);
154
} else if (UseAVX >= 1) {
155
__ vmovdqu(reg, Address(rsp, offset));
158
__ movdqu(reg, Address(rsp, offset));
165
const Address mxcsr_save(rsp, offset);
166
__ ldmxcsr(mxcsr_save);
169
__ block_comment("} restore_callee_saved_regs ");
172
static const int upcall_stub_code_base_size = 1024;
173
static const int upcall_stub_size_per_arg = 16;
175
address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
176
BasicType* out_sig_bt, int total_out_args,
178
jobject jabi, jobject jconv,
179
bool needs_return_buffer, int ret_buf_size) {
180
const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi);
181
const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv);
182
int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg);
183
CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 1);
184
if (buffer.blob() == nullptr) {
188
GrowableArray<VMStorage> unfiltered_out_regs;
189
int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs);
190
int preserved_bytes = SharedRuntime::out_preserve_stack_slots() * VMRegImpl::stack_slot_size;
191
int stack_bytes = preserved_bytes + out_arg_bytes;
192
int out_arg_area = align_up(stack_bytes , StackAlignmentInBytes);
194
// out_arg_area (for stack arguments) doubles as shadow space for native calls.
195
// make sure it is big enough.
196
if (out_arg_area < frame::arg_reg_save_area_bytes) {
197
out_arg_area = frame::arg_reg_save_area_bytes;
200
int reg_save_area_size = compute_reg_save_area_size(abi);
201
RegSpiller arg_spiller(call_regs._arg_regs);
202
RegSpiller result_spiller(call_regs._ret_regs);
204
int shuffle_area_offset = 0;
205
int res_save_area_offset = shuffle_area_offset + out_arg_area;
206
int arg_save_area_offset = res_save_area_offset + result_spiller.spill_size_bytes();
207
int reg_save_area_offset = arg_save_area_offset + arg_spiller.spill_size_bytes();
208
int frame_data_offset = reg_save_area_offset + reg_save_area_size;
209
int frame_bottom_offset = frame_data_offset + sizeof(UpcallStub::FrameData);
212
int ret_buf_offset = -1;
213
if (needs_return_buffer) {
214
ret_buf_offset = frame_bottom_offset;
215
frame_bottom_offset += ret_buf_size;
216
// use a free register for shuffling code to pick up return
217
// buffer address from
218
locs.set(StubLocations::RETURN_BUFFER, abi._scratch1);
221
VMStorage shuffle_reg = as_VMStorage(rbx);
222
GrowableArray<VMStorage> in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs);
223
GrowableArray<VMStorage> filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs);
224
ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, shuffle_reg);
227
LogTarget(Trace, foreign, upcall) lt;
228
if (lt.is_enabled()) {
231
arg_shuffle.print_on(&ls);
235
int frame_size = frame_bottom_offset;
236
frame_size = align_up(frame_size, StackAlignmentInBytes);
238
// Ok The space we have allocated will look like:
242
// |---------------------| = frame_bottom_offset = frame_size
245
// |---------------------| = ret_buf_offset
248
// |---------------------| = frame_data_offset
251
// |---------------------| = reg_save_are_offset
254
// |---------------------| = arg_save_are_offset
257
// |---------------------| = res_save_are_offset
259
// SP-> | out_arg_area | needs to be at end for shadow space
263
//////////////////////////////////////////////////////////////////////////////
265
MacroAssembler* _masm = new MacroAssembler(&buffer);
266
address start = __ pc();
267
__ enter(); // set up frame
268
if ((abi._stack_alignment_bytes % 16) != 0) {
269
// stack alignment of caller is not a multiple of 16
270
__ andptr(rsp, -StackAlignmentInBytes); // align stack
272
// allocate frame (frame_size is also aligned, so stack is still aligned)
273
__ subptr(rsp, frame_size);
275
// we have to always spill args since we need to do a call to get the thread
276
// (and maybe attach it).
277
arg_spiller.generate_spill(_masm, arg_save_area_offset);
279
preserve_callee_saved_registers(_masm, abi, reg_save_area_offset);
281
__ block_comment("{ on_entry");
283
__ lea(c_rarg0, Address(rsp, frame_data_offset));
284
__ movptr(c_rarg1, (intptr_t)receiver);
285
// stack already aligned
286
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::on_entry)));
287
__ movptr(r15_thread, rax);
288
__ reinit_heapbase();
289
__ block_comment("} on_entry");
291
__ block_comment("{ argument shuffle");
292
arg_spiller.generate_fill(_masm, arg_save_area_offset);
293
if (needs_return_buffer) {
294
assert(ret_buf_offset != -1, "no return buffer allocated");
295
__ lea(as_Register(locs.get(StubLocations::RETURN_BUFFER)), Address(rsp, ret_buf_offset));
297
arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, 0);
298
__ block_comment("} argument shuffle");
300
__ block_comment("{ receiver ");
301
__ get_vm_result(j_rarg0, r15_thread);
302
__ block_comment("} receiver ");
304
__ mov_metadata(rbx, entry);
305
__ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx); // just in case callee is deoptimized
307
__ push_cont_fastpath();
309
__ call(Address(rbx, Method::from_compiled_offset()));
311
__ pop_cont_fastpath();
313
// return value shuffle
314
if (!needs_return_buffer) {
316
if (call_regs._ret_regs.length() == 1) { // 0 or 1
317
VMStorage j_expected_result_reg;
325
j_expected_result_reg = as_VMStorage(rax);
329
j_expected_result_reg = as_VMStorage(xmm0);
332
fatal("unexpected return type: %s", type2name(ret_type));
334
// No need to move for now, since CallArranger can pick a return type
335
// that goes in the same reg for both CCs. But, at least assert they are the same
336
assert(call_regs._ret_regs.at(0) == j_expected_result_reg, "unexpected result register");
340
assert(ret_buf_offset != -1, "no return buffer allocated");
341
__ lea(rscratch1, Address(rsp, ret_buf_offset));
343
for (int i = 0; i < call_regs._ret_regs.length(); i++) {
344
VMStorage reg = call_regs._ret_regs.at(i);
345
if (reg.type() == StorageType::INTEGER) {
346
__ movptr(as_Register(reg), Address(rscratch1, offset));
348
} else if (reg.type() == StorageType::VECTOR) {
349
__ movdqu(as_XMMRegister(reg), Address(rscratch1, offset));
352
ShouldNotReachHere();
357
result_spiller.generate_spill(_masm, res_save_area_offset);
359
__ block_comment("{ on_exit");
361
__ lea(c_rarg0, Address(rsp, frame_data_offset));
362
// stack already aligned
363
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::on_exit)));
364
__ reinit_heapbase();
365
__ block_comment("} on_exit");
367
restore_callee_saved_registers(_masm, abi, reg_save_area_offset);
369
result_spiller.generate_fill(_masm, res_save_area_offset);
374
//////////////////////////////////////////////////////////////////////////////
380
ss.print("upcall_stub_%s", entry->signature()->as_C_string());
381
const char* name = _masm->code_string(ss.freeze());
383
const char* name = "upcall_stub";
386
buffer.log_section_sizes(name);
389
= UpcallStub::create(name,
392
in_ByteSize(frame_data_offset));
393
if (blob == nullptr) {
398
if (lt.is_enabled()) {
405
return blob->code_begin();