jdk

1
/*
2
 * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4
 *
5
 * This code is free software; you can redistribute it and/or modify it
6
 * under the terms of the GNU General Public License version 2 only, as
7
 * published by the Free Software Foundation.
8
 *
9
 * This code is distributed in the hope that it will be useful, but WITHOUT
10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12
 * version 2 for more details (a copy is included in the LICENSE file that
13
 * accompanied this code).
14
 *
15
 * You should have received a copy of the GNU General Public License version
16
 * 2 along with this work; if not, write to the Free Software Foundation,
17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18
 *
19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20
 * or visit www.oracle.com if you need additional information or have any
21
 * questions.
22
 *
23
 */
24

25
#include "precompiled.hpp"
26
#include "asm/codeBuffer.hpp"
27
#include "asm/macroAssembler.inline.hpp"
28
#include "memory/resourceArea.hpp"
29
#include "oops/access.inline.hpp"
30
#include "oops/klass.hpp"
31
#include "oops/oop.inline.hpp"
32
#include "prims/vectorSupport.hpp"
33
#include "runtime/continuation.hpp"
34
#include "runtime/interfaceSupport.inline.hpp"
35
#include "runtime/timerTrace.hpp"
36
#include "runtime/sharedRuntime.hpp"
37
#include "runtime/stubRoutines.hpp"
38
#include "utilities/align.hpp"
39
#include "utilities/copy.hpp"
40
#ifdef COMPILER2
41
#include "opto/runtime.hpp"
42
#endif
43

44
UnsafeMemoryAccess* UnsafeMemoryAccess::_table                  = nullptr;
45
int UnsafeMemoryAccess::_table_length                           = 0;
46
int UnsafeMemoryAccess::_table_max_length                       = 0;
47
address UnsafeMemoryAccess::_common_exit_stub_pc                = nullptr;
48

49
// Implementation of StubRoutines - for a description
50
// of how to extend it, see the header file.
51

52
// Class Variables
53

54
BufferBlob* StubRoutines::_initial_stubs_code                   = nullptr;
55
BufferBlob* StubRoutines::_final_stubs_code                     = nullptr;
56
BufferBlob* StubRoutines::_compiler_stubs_code                  = nullptr;
57
BufferBlob* StubRoutines::_continuation_stubs_code              = nullptr;
58

59
address StubRoutines::_call_stub_return_address                 = nullptr;
60
address StubRoutines::_call_stub_entry                          = nullptr;
61

62
address StubRoutines::_catch_exception_entry                    = nullptr;
63
address StubRoutines::_forward_exception_entry                  = nullptr;
64
address StubRoutines::_throw_AbstractMethodError_entry          = nullptr;
65
address StubRoutines::_throw_IncompatibleClassChangeError_entry = nullptr;
66
address StubRoutines::_throw_NullPointerException_at_call_entry = nullptr;
67
address StubRoutines::_throw_StackOverflowError_entry           = nullptr;
68
address StubRoutines::_throw_delayed_StackOverflowError_entry   = nullptr;
69
jint    StubRoutines::_verify_oop_count                         = 0;
70
address StubRoutines::_verify_oop_subroutine_entry              = nullptr;
71
address StubRoutines::_atomic_xchg_entry                        = nullptr;
72
address StubRoutines::_atomic_cmpxchg_entry                     = nullptr;
73
address StubRoutines::_atomic_cmpxchg_long_entry                = nullptr;
74
address StubRoutines::_atomic_add_entry                         = nullptr;
75
address StubRoutines::_fence_entry                              = nullptr;
76

77
// Compiled code entry points default values
78
// The default functions don't have separate disjoint versions.
79
address StubRoutines::_jbyte_arraycopy          = CAST_FROM_FN_PTR(address, StubRoutines::jbyte_copy);
80
address StubRoutines::_jshort_arraycopy         = CAST_FROM_FN_PTR(address, StubRoutines::jshort_copy);
81
address StubRoutines::_jint_arraycopy           = CAST_FROM_FN_PTR(address, StubRoutines::jint_copy);
82
address StubRoutines::_jlong_arraycopy          = CAST_FROM_FN_PTR(address, StubRoutines::jlong_copy);
83
address StubRoutines::_oop_arraycopy            = CAST_FROM_FN_PTR(address, StubRoutines::oop_copy);
84
address StubRoutines::_oop_arraycopy_uninit     = CAST_FROM_FN_PTR(address, StubRoutines::oop_copy_uninit);
85
address StubRoutines::_jbyte_disjoint_arraycopy          = CAST_FROM_FN_PTR(address, StubRoutines::jbyte_copy);
86
address StubRoutines::_jshort_disjoint_arraycopy         = CAST_FROM_FN_PTR(address, StubRoutines::jshort_copy);
87
address StubRoutines::_jint_disjoint_arraycopy           = CAST_FROM_FN_PTR(address, StubRoutines::jint_copy);
88
address StubRoutines::_jlong_disjoint_arraycopy          = CAST_FROM_FN_PTR(address, StubRoutines::jlong_copy);
89
address StubRoutines::_oop_disjoint_arraycopy            = CAST_FROM_FN_PTR(address, StubRoutines::oop_copy);
90
address StubRoutines::_oop_disjoint_arraycopy_uninit     = CAST_FROM_FN_PTR(address, StubRoutines::oop_copy_uninit);
91

92
address StubRoutines::_arrayof_jbyte_arraycopy  = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jbyte_copy);
93
address StubRoutines::_arrayof_jshort_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jshort_copy);
94
address StubRoutines::_arrayof_jint_arraycopy   = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jint_copy);
95
address StubRoutines::_arrayof_jlong_arraycopy  = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jlong_copy);
96
address StubRoutines::_arrayof_oop_arraycopy    = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_oop_copy);
97
address StubRoutines::_arrayof_oop_arraycopy_uninit      = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_oop_copy_uninit);
98
address StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jbyte_copy);
99
address StubRoutines::_arrayof_jshort_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jshort_copy);
100
address StubRoutines::_arrayof_jint_disjoint_arraycopy   = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jint_copy);
101
address StubRoutines::_arrayof_jlong_disjoint_arraycopy  = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jlong_copy);
102
address StubRoutines::_arrayof_oop_disjoint_arraycopy    = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_oop_copy);
103
address StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit  = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_oop_copy_uninit);
104

105
address StubRoutines::_data_cache_writeback              = nullptr;
106
address StubRoutines::_data_cache_writeback_sync         = nullptr;
107

108
address StubRoutines::_checkcast_arraycopy               = nullptr;
109
address StubRoutines::_checkcast_arraycopy_uninit        = nullptr;
110
address StubRoutines::_unsafe_arraycopy                  = nullptr;
111
address StubRoutines::_generic_arraycopy                 = nullptr;
112

113
address StubRoutines::_unsafe_setmemory                  = nullptr;
114

115
address StubRoutines::_jbyte_fill;
116
address StubRoutines::_jshort_fill;
117
address StubRoutines::_jint_fill;
118
address StubRoutines::_arrayof_jbyte_fill;
119
address StubRoutines::_arrayof_jshort_fill;
120
address StubRoutines::_arrayof_jint_fill;
121

122
address StubRoutines::_aescrypt_encryptBlock               = nullptr;
123
address StubRoutines::_aescrypt_decryptBlock               = nullptr;
124
address StubRoutines::_cipherBlockChaining_encryptAESCrypt = nullptr;
125
address StubRoutines::_cipherBlockChaining_decryptAESCrypt = nullptr;
126
address StubRoutines::_electronicCodeBook_encryptAESCrypt  = nullptr;
127
address StubRoutines::_electronicCodeBook_decryptAESCrypt  = nullptr;
128
address StubRoutines::_counterMode_AESCrypt                = nullptr;
129
address StubRoutines::_galoisCounterMode_AESCrypt          = nullptr;
130
address StubRoutines::_ghash_processBlocks                 = nullptr;
131
address StubRoutines::_chacha20Block                       = nullptr;
132
address StubRoutines::_base64_encodeBlock                  = nullptr;
133
address StubRoutines::_base64_decodeBlock                  = nullptr;
134
address StubRoutines::_poly1305_processBlocks              = nullptr;
135
address StubRoutines::_intpoly_montgomeryMult_P256         = nullptr;
136
address StubRoutines::_intpoly_assign                      = nullptr;
137

138
address StubRoutines::_md5_implCompress      = nullptr;
139
address StubRoutines::_md5_implCompressMB    = nullptr;
140
address StubRoutines::_sha1_implCompress     = nullptr;
141
address StubRoutines::_sha1_implCompressMB   = nullptr;
142
address StubRoutines::_sha256_implCompress   = nullptr;
143
address StubRoutines::_sha256_implCompressMB = nullptr;
144
address StubRoutines::_sha512_implCompress   = nullptr;
145
address StubRoutines::_sha512_implCompressMB = nullptr;
146
address StubRoutines::_sha3_implCompress     = nullptr;
147
address StubRoutines::_sha3_implCompressMB   = nullptr;
148

149
address StubRoutines::_updateBytesCRC32 = nullptr;
150
address StubRoutines::_crc_table_adr =    nullptr;
151

152
address StubRoutines::_string_indexof_array[4]   =    { nullptr };
153

154
address StubRoutines::_crc32c_table_addr = nullptr;
155
address StubRoutines::_updateBytesCRC32C = nullptr;
156
address StubRoutines::_updateBytesAdler32 = nullptr;
157

158
address StubRoutines::_multiplyToLen = nullptr;
159
address StubRoutines::_squareToLen = nullptr;
160
address StubRoutines::_mulAdd = nullptr;
161
address StubRoutines::_montgomeryMultiply = nullptr;
162
address StubRoutines::_montgomerySquare = nullptr;
163
address StubRoutines::_bigIntegerRightShiftWorker = nullptr;
164
address StubRoutines::_bigIntegerLeftShiftWorker = nullptr;
165

166
address StubRoutines::_vectorizedMismatch = nullptr;
167

168
address StubRoutines::_dexp = nullptr;
169
address StubRoutines::_dlog = nullptr;
170
address StubRoutines::_dlog10 = nullptr;
171
address StubRoutines::_fmod = nullptr;
172
address StubRoutines::_dpow = nullptr;
173
address StubRoutines::_dsin = nullptr;
174
address StubRoutines::_dcos = nullptr;
175
address StubRoutines::_dlibm_sin_cos_huge = nullptr;
176
address StubRoutines::_dlibm_reduce_pi04l = nullptr;
177
address StubRoutines::_dlibm_tan_cot_huge = nullptr;
178
address StubRoutines::_dtan = nullptr;
179

180
address StubRoutines::_f2hf = nullptr;
181
address StubRoutines::_hf2f = nullptr;
182

183
address StubRoutines::_vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP] = {{nullptr}, {nullptr}};
184
address StubRoutines::_vector_d_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP] = {{nullptr}, {nullptr}};
185

186
address StubRoutines::_method_entry_barrier = nullptr;
187
address StubRoutines::_array_sort = nullptr;
188
address StubRoutines::_array_partition  = nullptr;
189

190
address StubRoutines::_cont_thaw          = nullptr;
191
address StubRoutines::_cont_returnBarrier = nullptr;
192
address StubRoutines::_cont_returnBarrierExc = nullptr;
193

194
JFR_ONLY(RuntimeStub* StubRoutines::_jfr_write_checkpoint_stub = nullptr;)
195
JFR_ONLY(address StubRoutines::_jfr_write_checkpoint = nullptr;)
196
JFR_ONLY(RuntimeStub* StubRoutines::_jfr_return_lease_stub = nullptr;)
197
JFR_ONLY(address StubRoutines::_jfr_return_lease = nullptr;)
198

199
address StubRoutines::_upcall_stub_exception_handler = nullptr;
200

201
address StubRoutines::_lookup_secondary_supers_table_slow_path_stub = nullptr;
202
address StubRoutines::_lookup_secondary_supers_table_stubs[Klass::SECONDARY_SUPERS_TABLE_SIZE] = { nullptr };
203

204

205
// Initialization
206
//
207
// Note: to break cycle with universe initialization, stubs are generated in two phases.
208
// The first one generates stubs needed during universe init (e.g., _handle_must_compile_first_entry).
209
// The second phase includes all other stubs (which may depend on universe being initialized.)
210

211
extern void StubGenerator_generate(CodeBuffer* code, StubCodeGenerator::StubsKind kind); // only interface to generators
212

213
void UnsafeMemoryAccess::create_table(int max_size) {
214
  UnsafeMemoryAccess::_table = new UnsafeMemoryAccess[max_size];
215
  UnsafeMemoryAccess::_table_max_length = max_size;
216
}
217

218
bool UnsafeMemoryAccess::contains_pc(address pc) {
219
  for (int i = 0; i < UnsafeMemoryAccess::_table_length; i++) {
220
    UnsafeMemoryAccess* entry = &UnsafeMemoryAccess::_table[i];
221
    if (pc >= entry->start_pc() && pc < entry->end_pc()) {
222
      return true;
223
    }
224
  }
225
  return false;
226
}
227

228
address UnsafeMemoryAccess::page_error_continue_pc(address pc) {
229
  for (int i = 0; i < UnsafeMemoryAccess::_table_length; i++) {
230
    UnsafeMemoryAccess* entry = &UnsafeMemoryAccess::_table[i];
231
    if (pc >= entry->start_pc() && pc < entry->end_pc()) {
232
      return entry->error_exit_pc();
233
    }
234
  }
235
  return nullptr;
236
}
237

238

239
static BufferBlob* initialize_stubs(StubCodeGenerator::StubsKind kind,
240
                                    int code_size, int max_aligned_stubs,
241
                                    const char* timer_msg,
242
                                    const char* buffer_name,
243
                                    const char* assert_msg) {
244
  ResourceMark rm;
245
  TraceTime timer(timer_msg, TRACETIME_LOG(Info, startuptime));
246
  // Add extra space for large CodeEntryAlignment
247
  int size = code_size + CodeEntryAlignment * max_aligned_stubs;
248
  BufferBlob* stubs_code = BufferBlob::create(buffer_name, size);
249
  if (stubs_code == nullptr) {
250
    vm_exit_out_of_memory(code_size, OOM_MALLOC_ERROR, "CodeCache: no room for %s", buffer_name);
251
  }
252
  CodeBuffer buffer(stubs_code);
253
  StubGenerator_generate(&buffer, kind);
254
  // When new stubs added we need to make sure there is some space left
255
  // to catch situation when we should increase size again.
256
  assert(code_size == 0 || buffer.insts_remaining() > 200, "increase %s", assert_msg);
257

258
  LogTarget(Info, stubs) lt;
259
  if (lt.is_enabled()) {
260
    LogStream ls(lt);
261
    ls.print_cr("%s\t [" INTPTR_FORMAT ", " INTPTR_FORMAT "] used: %d, free: %d",
262
                buffer_name, p2i(stubs_code->content_begin()), p2i(stubs_code->content_end()),
263
                buffer.total_content_size(), buffer.insts_remaining());
264
  }
265
  return stubs_code;
266
}
267

268
void StubRoutines::initialize_initial_stubs() {
269
  if (_initial_stubs_code == nullptr) {
270
    _initial_stubs_code = initialize_stubs(StubCodeGenerator::Initial_stubs,
271
                                           _initial_stubs_code_size, 10,
272
                                           "StubRoutines generation initial stubs",
273
                                           "StubRoutines (initial stubs)",
274
                                           "_initial_stubs_code_size");
275
  }
276
}
277

278
void StubRoutines::initialize_continuation_stubs() {
279
  if (_continuation_stubs_code == nullptr) {
280
    _continuation_stubs_code = initialize_stubs(StubCodeGenerator::Continuation_stubs,
281
                                           _continuation_stubs_code_size, 10,
282
                                           "StubRoutines generation continuation stubs",
283
                                           "StubRoutines (continuation stubs)",
284
                                           "_continuation_stubs_code_size");
285
  }
286
}
287

288
void StubRoutines::initialize_compiler_stubs() {
289
  if (_compiler_stubs_code == nullptr) {
290
    _compiler_stubs_code = initialize_stubs(StubCodeGenerator::Compiler_stubs,
291
                                           _compiler_stubs_code_size, 100,
292
                                           "StubRoutines generation compiler stubs",
293
                                           "StubRoutines (compiler stubs)",
294
                                           "_compiler_stubs_code_size");
295
  }
296
}
297

298
void StubRoutines::initialize_final_stubs() {
299
  if (_final_stubs_code == nullptr) {
300
    _final_stubs_code = initialize_stubs(StubCodeGenerator::Final_stubs,
301
                                         _final_stubs_code_size, 10,
302
                                         "StubRoutines generation final stubs",
303
                                         "StubRoutines (final stubs)",
304
                                         "_final_stubs_code_size");
305
  }
306
}
307

308
void initial_stubs_init()      { StubRoutines::initialize_initial_stubs(); }
309
void continuation_stubs_init() { StubRoutines::initialize_continuation_stubs(); }
310
void final_stubs_init()        { StubRoutines::initialize_final_stubs(); }
311

312
void compiler_stubs_init(bool in_compiler_thread) {
313
  if (in_compiler_thread && DelayCompilerStubsGeneration) {
314
    // Temporarily revert state of stubs generation because
315
    // it is called after final_stubs_init() finished
316
    // during compiler runtime initialization.
317
    // It is fine because these stubs are only used by
318
    // compiled code and compiler is not running yet.
319
    StubCodeDesc::unfreeze();
320
    StubRoutines::initialize_compiler_stubs();
321
    StubCodeDesc::freeze();
322
  } else if (!in_compiler_thread && !DelayCompilerStubsGeneration) {
323
    StubRoutines::initialize_compiler_stubs();
324
  }
325
}
326

327
//
328
// Default versions of arraycopy functions
329
//
330

331
JRT_LEAF(void, StubRoutines::jbyte_copy(jbyte* src, jbyte* dest, size_t count))
332
#ifndef PRODUCT
333
  SharedRuntime::_jbyte_array_copy_ctr++;      // Slow-path byte array copy
334
#endif // !PRODUCT
335
  Copy::conjoint_jbytes_atomic(src, dest, count);
336
JRT_END
337

338
JRT_LEAF(void, StubRoutines::jshort_copy(jshort* src, jshort* dest, size_t count))
339
#ifndef PRODUCT
340
  SharedRuntime::_jshort_array_copy_ctr++;     // Slow-path short/char array copy
341
#endif // !PRODUCT
342
  Copy::conjoint_jshorts_atomic(src, dest, count);
343
JRT_END
344

345
JRT_LEAF(void, StubRoutines::jint_copy(jint* src, jint* dest, size_t count))
346
#ifndef PRODUCT
347
  SharedRuntime::_jint_array_copy_ctr++;       // Slow-path int/float array copy
348
#endif // !PRODUCT
349
  Copy::conjoint_jints_atomic(src, dest, count);
350
JRT_END
351

352
JRT_LEAF(void, StubRoutines::jlong_copy(jlong* src, jlong* dest, size_t count))
353
#ifndef PRODUCT
354
  SharedRuntime::_jlong_array_copy_ctr++;      // Slow-path long/double array copy
355
#endif // !PRODUCT
356
  Copy::conjoint_jlongs_atomic(src, dest, count);
357
JRT_END
358

359
JRT_LEAF(void, StubRoutines::oop_copy(oop* src, oop* dest, size_t count))
360
#ifndef PRODUCT
361
  SharedRuntime::_oop_array_copy_ctr++;        // Slow-path oop array copy
362
#endif // !PRODUCT
363
  assert(count != 0, "count should be non-zero");
364
  ArrayAccess<>::oop_arraycopy_raw((HeapWord*)src, (HeapWord*)dest, count);
365
JRT_END
366

367
JRT_LEAF(void, StubRoutines::oop_copy_uninit(oop* src, oop* dest, size_t count))
368
#ifndef PRODUCT
369
  SharedRuntime::_oop_array_copy_ctr++;        // Slow-path oop array copy
370
#endif // !PRODUCT
371
  assert(count != 0, "count should be non-zero");
372
  ArrayAccess<IS_DEST_UNINITIALIZED>::oop_arraycopy_raw((HeapWord*)src, (HeapWord*)dest, count);
373
JRT_END
374

375
JRT_LEAF(void, StubRoutines::arrayof_jbyte_copy(HeapWord* src, HeapWord* dest, size_t count))
376
#ifndef PRODUCT
377
  SharedRuntime::_jbyte_array_copy_ctr++;      // Slow-path byte array copy
378
#endif // !PRODUCT
379
  Copy::arrayof_conjoint_jbytes(src, dest, count);
380
JRT_END
381

382
JRT_LEAF(void, StubRoutines::arrayof_jshort_copy(HeapWord* src, HeapWord* dest, size_t count))
383
#ifndef PRODUCT
384
  SharedRuntime::_jshort_array_copy_ctr++;     // Slow-path short/char array copy
385
#endif // !PRODUCT
386
  Copy::arrayof_conjoint_jshorts(src, dest, count);
387
JRT_END
388

389
JRT_LEAF(void, StubRoutines::arrayof_jint_copy(HeapWord* src, HeapWord* dest, size_t count))
390
#ifndef PRODUCT
391
  SharedRuntime::_jint_array_copy_ctr++;       // Slow-path int/float array copy
392
#endif // !PRODUCT
393
  Copy::arrayof_conjoint_jints(src, dest, count);
394
JRT_END
395

396
JRT_LEAF(void, StubRoutines::arrayof_jlong_copy(HeapWord* src, HeapWord* dest, size_t count))
397
#ifndef PRODUCT
398
  SharedRuntime::_jlong_array_copy_ctr++;       // Slow-path int/float array copy
399
#endif // !PRODUCT
400
  Copy::arrayof_conjoint_jlongs(src, dest, count);
401
JRT_END
402

403
JRT_LEAF(void, StubRoutines::arrayof_oop_copy(HeapWord* src, HeapWord* dest, size_t count))
404
#ifndef PRODUCT
405
  SharedRuntime::_oop_array_copy_ctr++;        // Slow-path oop array copy
406
#endif // !PRODUCT
407
  assert(count != 0, "count should be non-zero");
408
  ArrayAccess<ARRAYCOPY_ARRAYOF>::oop_arraycopy_raw(src, dest, count);
409
JRT_END
410

411
JRT_LEAF(void, StubRoutines::arrayof_oop_copy_uninit(HeapWord* src, HeapWord* dest, size_t count))
412
#ifndef PRODUCT
413
  SharedRuntime::_oop_array_copy_ctr++;        // Slow-path oop array copy
414
#endif // !PRODUCT
415
  assert(count != 0, "count should be non-zero");
416
  ArrayAccess<ARRAYCOPY_ARRAYOF | IS_DEST_UNINITIALIZED>::oop_arraycopy_raw(src, dest, count);
417
JRT_END
418

419
address StubRoutines::select_fill_function(BasicType t, bool aligned, const char* &name) {
420
#define RETURN_STUB(xxx_fill) { \
421
  name = #xxx_fill; \
422
  return StubRoutines::xxx_fill(); }
423

424
  switch (t) {
425
  case T_BYTE:
426
  case T_BOOLEAN:
427
    if (!aligned) RETURN_STUB(jbyte_fill);
428
    RETURN_STUB(arrayof_jbyte_fill);
429
  case T_CHAR:
430
  case T_SHORT:
431
    if (!aligned) RETURN_STUB(jshort_fill);
432
    RETURN_STUB(arrayof_jshort_fill);
433
  case T_INT:
434
  case T_FLOAT:
435
    if (!aligned) RETURN_STUB(jint_fill);
436
    RETURN_STUB(arrayof_jint_fill);
437
  case T_DOUBLE:
438
  case T_LONG:
439
  case T_ARRAY:
440
  case T_OBJECT:
441
  case T_NARROWOOP:
442
  case T_NARROWKLASS:
443
  case T_ADDRESS:
444
  case T_VOID:
445
    // Currently unsupported
446
    return nullptr;
447

448
  default:
449
    ShouldNotReachHere();
450
    return nullptr;
451
  }
452

453
#undef RETURN_STUB
454
}
455

456
// constants for computing the copy function
457
enum {
458
  COPYFUNC_UNALIGNED = 0,
459
  COPYFUNC_ALIGNED = 1,                 // src, dest aligned to HeapWordSize
460
  COPYFUNC_CONJOINT = 0,
461
  COPYFUNC_DISJOINT = 2                 // src != dest, or transfer can descend
462
};
463

464
// Note:  The condition "disjoint" applies also for overlapping copies
465
// where an descending copy is permitted (i.e., dest_offset <= src_offset).
466
address
467
StubRoutines::select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name, bool dest_uninitialized) {
468
  int selector =
469
    (aligned  ? COPYFUNC_ALIGNED  : COPYFUNC_UNALIGNED) +
470
    (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT);
471

472
#define RETURN_STUB(xxx_arraycopy) { \
473
  name = #xxx_arraycopy; \
474
  return StubRoutines::xxx_arraycopy(); }
475

476
#define RETURN_STUB_PARM(xxx_arraycopy, parm) { \
477
  name = parm ? #xxx_arraycopy "_uninit": #xxx_arraycopy; \
478
  return StubRoutines::xxx_arraycopy(parm); }
479

480
  switch (t) {
481
  case T_BYTE:
482
  case T_BOOLEAN:
483
    switch (selector) {
484
    case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jbyte_arraycopy);
485
    case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jbyte_arraycopy);
486
    case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jbyte_disjoint_arraycopy);
487
    case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jbyte_disjoint_arraycopy);
488
    }
489
  case T_CHAR:
490
  case T_SHORT:
491
    switch (selector) {
492
    case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jshort_arraycopy);
493
    case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jshort_arraycopy);
494
    case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jshort_disjoint_arraycopy);
495
    case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jshort_disjoint_arraycopy);
496
    }
497
  case T_INT:
498
  case T_FLOAT:
499
    switch (selector) {
500
    case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jint_arraycopy);
501
    case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jint_arraycopy);
502
    case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jint_disjoint_arraycopy);
503
    case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jint_disjoint_arraycopy);
504
    }
505
  case T_DOUBLE:
506
  case T_LONG:
507
    switch (selector) {
508
    case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jlong_arraycopy);
509
    case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jlong_arraycopy);
510
    case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jlong_disjoint_arraycopy);
511
    case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jlong_disjoint_arraycopy);
512
    }
513
  case T_ARRAY:
514
  case T_OBJECT:
515
    switch (selector) {
516
    case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB_PARM(oop_arraycopy, dest_uninitialized);
517
    case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB_PARM(arrayof_oop_arraycopy, dest_uninitialized);
518
    case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB_PARM(oop_disjoint_arraycopy, dest_uninitialized);
519
    case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB_PARM(arrayof_oop_disjoint_arraycopy, dest_uninitialized);
520
    }
521
  default:
522
    ShouldNotReachHere();
523
    return nullptr;
524
  }
525

526
#undef RETURN_STUB
527
#undef RETURN_STUB_PARM
528
}
529

530
UnsafeMemoryAccessMark::UnsafeMemoryAccessMark(StubCodeGenerator* cgen, bool add_entry, bool continue_at_scope_end, address error_exit_pc) {
531
  _cgen = cgen;
532
  _ucm_entry = nullptr;
533
  if (add_entry) {
534
    address err_exit_pc = nullptr;
535
    if (!continue_at_scope_end) {
536
      err_exit_pc = error_exit_pc != nullptr ? error_exit_pc : UnsafeMemoryAccess::common_exit_stub_pc();
537
    }
538
    assert(err_exit_pc != nullptr || continue_at_scope_end, "error exit not set");
539
    _ucm_entry = UnsafeMemoryAccess::add_to_table(_cgen->assembler()->pc(), nullptr, err_exit_pc);
540
  }
541
}
542

543
UnsafeMemoryAccessMark::~UnsafeMemoryAccessMark() {
544
  if (_ucm_entry != nullptr) {
545
    _ucm_entry->set_end_pc(_cgen->assembler()->pc());
546
    if (_ucm_entry->error_exit_pc() == nullptr) {
547
      _ucm_entry->set_error_exit_pc(_cgen->assembler()->pc());
548
    }
549
  }
550
}
551