2
* Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
3
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5
* This code is free software; you can redistribute it and/or modify it
6
* under the terms of the GNU General Public License version 2 only, as
7
* published by the Free Software Foundation.
9
* This code is distributed in the hope that it will be useful, but WITHOUT
10
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12
* version 2 for more details (a copy is included in the LICENSE file that
13
* accompanied this code).
15
* You should have received a copy of the GNU General Public License version
16
* 2 along with this work; if not, write to the Free Software Foundation,
17
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20
* or visit www.oracle.com if you need additional information or have any
25
#include "precompiled.hpp"
26
#include "classfile/javaClasses.inline.hpp"
27
#include "classfile/vmSymbols.hpp"
28
#include "code/codeCache.inline.hpp"
29
#include "code/nmethod.inline.hpp"
30
#include "code/vmreg.inline.hpp"
31
#include "compiler/oopMap.inline.hpp"
32
#include "gc/shared/continuationGCSupport.inline.hpp"
33
#include "gc/shared/gc_globals.hpp"
34
#include "gc/shared/barrierSet.hpp"
35
#include "gc/shared/memAllocator.hpp"
36
#include "gc/shared/threadLocalAllocBuffer.inline.hpp"
37
#include "interpreter/interpreter.hpp"
38
#include "jfr/jfrEvents.hpp"
39
#include "logging/log.hpp"
40
#include "logging/logStream.hpp"
41
#include "oops/access.inline.hpp"
42
#include "oops/method.inline.hpp"
43
#include "oops/oopsHierarchy.hpp"
44
#include "oops/objArrayOop.inline.hpp"
45
#include "oops/stackChunkOop.inline.hpp"
46
#include "prims/jvmtiThreadState.hpp"
47
#include "runtime/arguments.hpp"
48
#include "runtime/continuation.hpp"
49
#include "runtime/continuationEntry.inline.hpp"
50
#include "runtime/continuationHelper.inline.hpp"
51
#include "runtime/continuationJavaClasses.inline.hpp"
52
#include "runtime/continuationWrapper.inline.hpp"
53
#include "runtime/frame.inline.hpp"
54
#include "runtime/interfaceSupport.inline.hpp"
55
#include "runtime/javaThread.inline.hpp"
56
#include "runtime/jniHandles.inline.hpp"
57
#include "runtime/keepStackGCProcessed.hpp"
58
#include "runtime/orderAccess.hpp"
59
#include "runtime/prefetch.inline.hpp"
60
#include "runtime/smallRegisterMap.inline.hpp"
61
#include "runtime/sharedRuntime.hpp"
62
#include "runtime/stackChunkFrameStream.inline.hpp"
63
#include "runtime/stackFrameStream.inline.hpp"
64
#include "runtime/stackOverflow.hpp"
65
#include "runtime/stackWatermarkSet.inline.hpp"
66
#include "utilities/debug.hpp"
67
#include "utilities/exceptions.hpp"
68
#include "utilities/macros.hpp"
70
#include "gc/z/zStackChunkGCData.inline.hpp"
76
* This file contains the implementation of continuation freezing (yield) and thawing (run).
78
* This code is very latency-critical and very hot. An ordinary and well-behaved server application
79
* would likely call these operations many thousands of times per second second, on every core.
81
* Freeze might be called every time the application performs any I/O operation, every time it
82
* acquires a j.u.c. lock, every time it takes a message from a queue, and thaw can be called
83
* multiple times in each of those cases, as it is called by the return barrier, which may be
84
* invoked on method return.
86
* The amortized budget for each of those two operations is ~100-150ns. That is why, for
87
* example, every effort is made to avoid Java-VM transitions as much as possible.
89
* On the fast path, all frames are known to be compiled, and the chunk requires no barriers
90
* and so frames simply copied, and the bottom-most one is patched.
91
* On the slow path, internal pointers in interpreted frames are de/relativized to/from offsets
92
* and absolute pointers, and barriers invoked.
95
/************************************************
97
Thread-stack layout on freeze/thaw.
98
See corresponding stack-chunk layout in instanceStackChunkKlass.hpp
100
+----------------------------+
106
|----------------------------|
110
|============================|
111
| enterSpecial frame |
115
^ | int argsize | = ContinuationEntry
118
| | ContinuationEntry* parent |
120
| |============================| <------ JavaThread::_cont_entry = entry->sp()
121
| | ? alignment word ? |
122
| |----------------------------| <--\
124
| | ? caller stack args ? | | argsize (might not be 2-word aligned) words
125
Address | | | | Caller is still in the chunk.
126
| |----------------------------| |
127
| | pc (? return barrier ?) | | This pc contains the return barrier when the bottom-most frame
128
| | rbp | | isn't the last one in the continuation.
132
+----------------------------| \__ Continuation frames to be frozen/thawed
136
|----------------------------| |
140
|----------------------------| <--/
142
| doYield/safepoint stub | When preempting forcefully, we could have a safepoint stub
143
| | instead of a doYield stub
144
|============================| <- the sp passed to freeze
146
| Native freeze/thaw frames |
150
+----------------------------+
152
************************************************/
154
static const bool TEST_THAW_ONE_CHUNK_FRAME = false; // force thawing frames one-at-a-time for testing
156
#define CONT_JFR false // emit low-level JFR events that count slow/fast path for continuation performance debugging only
158
#define CONT_JFR_ONLY(code) code
160
#define CONT_JFR_ONLY(code)
163
// TODO: See AbstractAssembler::generate_stack_overflow_check,
164
// Compile::bang_size_in_bytes(), m->as_SafePoint()->jvms()->interpreter_frame_size()
165
// when we stack-bang, we need to update a thread field with the lowest (farthest) bang point.
167
// Data invariants are defined by Continuation::debug_verify_continuation and Continuation::debug_verify_stack_chunk
169
// Used to just annotatate cold/hot branches
170
#define LIKELY(condition) (condition)
171
#define UNLIKELY(condition) (condition)
173
// debugging functions
175
extern "C" bool dbg_is_safe(const void* p, intptr_t errvalue); // address p is readable and *(intptr_t*)p != errvalue
177
static void verify_continuation(oop continuation) { Continuation::debug_verify_continuation(continuation); }
179
static void do_deopt_after_thaw(JavaThread* thread);
180
static bool do_verify_after_thaw(JavaThread* thread, stackChunkOop chunk, outputStream* st);
181
static void log_frames(JavaThread* thread);
182
static void print_frame_layout(const frame& f, bool callee_complete, outputStream* st = tty);
184
#define assert_pfl(p, ...) \
187
JavaThread* t = JavaThread::active(); \
188
if (t->has_last_Java_frame()) { \
189
tty->print_cr("assert(" #p ") failed:"); \
190
t->print_frame_layout(); \
193
vmassert(p, __VA_ARGS__); \
197
static void verify_continuation(oop continuation) { }
198
#define assert_pfl(p, ...)
201
// should match Continuation.preemptStatus() in Continuation.java
204
freeze_ok_bottom = 1,
205
freeze_pinned_cs = 2,
206
freeze_pinned_native = 3,
207
freeze_pinned_monitor = 4,
211
const char* freeze_result_names[6] = {
215
"freeze_pinned_native",
216
"freeze_pinned_monitor",
220
static freeze_result is_pinned0(JavaThread* thread, oop cont_scope, bool safepoint);
221
template<typename ConfigT> static inline int freeze_internal(JavaThread* current, intptr_t* const sp);
223
static inline int prepare_thaw_internal(JavaThread* thread, bool return_barrier);
224
template<typename ConfigT> static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::thaw_kind kind);
227
// Entry point to freeze. Transitions are handled manually
228
// Called from gen_continuation_yield() in sharedRuntime_<cpu>.cpp through Continuation::freeze_entry();
229
template<typename ConfigT>
230
static JRT_BLOCK_ENTRY(int, freeze(JavaThread* current, intptr_t* sp))
231
assert(sp == current->frame_anchor()->last_Java_sp(), "");
233
if (current->raw_cont_fastpath() > current->last_continuation()->entry_sp() || current->raw_cont_fastpath() < sp) {
234
current->set_cont_fastpath(nullptr);
237
return ConfigT::freeze(current, sp);
240
JRT_LEAF(int, Continuation::prepare_thaw(JavaThread* thread, bool return_barrier))
241
return prepare_thaw_internal(thread, return_barrier);
244
template<typename ConfigT>
245
static JRT_LEAF(intptr_t*, thaw(JavaThread* thread, int kind))
246
// TODO: JRT_LEAF and NoHandleMark is problematic for JFR events.
247
// vFrameStreamCommon allocates Handles in RegisterMap for continuations.
248
// JRT_ENTRY instead?
249
ResetNoHandleMark rnhm;
251
// we might modify the code cache via BarrierSetNMethod::nmethod_entry_barrier
252
MACOS_AARCH64_ONLY(ThreadWXEnable __wx(WXWrite, thread));
253
return ConfigT::thaw(thread, (Continuation::thaw_kind)kind);
256
JVM_ENTRY(jint, CONT_isPinned0(JNIEnv* env, jobject cont_scope)) {
257
JavaThread* thread = JavaThread::thread_from_jni_environment(env);
258
return is_pinned0(thread, JNIHandles::resolve(cont_scope), false);
264
enum class oop_kind { NARROW, WIDE };
265
template <oop_kind oops, typename BarrierSetT>
268
typedef Config<oops, BarrierSetT> SelfT;
269
using OopT = std::conditional_t<oops == oop_kind::NARROW, narrowOop, oop>;
271
static int freeze(JavaThread* thread, intptr_t* const sp) {
272
return freeze_internal<SelfT>(thread, sp);
275
static intptr_t* thaw(JavaThread* thread, Continuation::thaw_kind kind) {
276
return thaw_internal<SelfT>(thread, kind);
280
static bool stack_overflow_check(JavaThread* thread, size_t size, address sp) {
281
const size_t page_size = os::vm_page_size();
282
if (size > page_size) {
283
if (sp - size < thread->stack_overflow_state()->shadow_zone_safe_limit()) {
291
static oop get_continuation(JavaThread* thread) {
292
assert(thread != nullptr, "");
293
assert(thread->threadObj() != nullptr, "");
294
return java_lang_Thread::continuation(thread->threadObj());
297
inline void clear_anchor(JavaThread* thread) {
298
thread->frame_anchor()->clear();
301
static void set_anchor(JavaThread* thread, intptr_t* sp) {
302
address pc = ContinuationHelper::return_address_at(
303
sp - frame::sender_sp_ret_address_offset());
304
assert(pc != nullptr, "");
306
JavaFrameAnchor* anchor = thread->frame_anchor();
307
anchor->set_last_Java_sp(sp);
308
anchor->set_last_Java_pc(pc);
309
ContinuationHelper::set_anchor_pd(anchor, sp);
311
assert(thread->has_last_Java_frame(), "");
312
assert(thread->last_frame().cb() != nullptr, "");
316
static void set_anchor_to_entry(JavaThread* thread, ContinuationEntry* entry) {
317
JavaFrameAnchor* anchor = thread->frame_anchor();
318
anchor->set_last_Java_sp(entry->entry_sp());
319
anchor->set_last_Java_pc(entry->entry_pc());
320
ContinuationHelper::set_anchor_to_entry_pd(anchor, entry);
322
assert(thread->has_last_Java_frame(), "");
323
assert(thread->last_frame().cb() != nullptr, "");
327
class FreezeThawJfrInfo : public StackObj {
329
short _e_num_interpreted_frames;
332
FreezeThawJfrInfo() : _e_size(0), _e_num_interpreted_frames(0) {}
333
inline void record_interpreted_frame() { _e_num_interpreted_frames++; }
334
inline void record_size_copied(int size) { _e_size += size << LogBytesPerWord; }
335
template<typename Event> void post_jfr_event(Event *e, oop continuation, JavaThread* jt);
338
template<typename Event> void FreezeThawJfrInfo::post_jfr_event(Event* e, oop continuation, JavaThread* jt) {
339
if (e->should_commit()) {
340
log_develop_trace(continuations)("JFR event: iframes: %d size: %d", _e_num_interpreted_frames, _e_size);
341
e->set_carrierThread(JFR_JVM_THREAD_ID(jt));
342
e->set_continuationClass(continuation->klass());
343
e->set_interpretedFrames(_e_num_interpreted_frames);
344
e->set_size(_e_size);
350
/////////////// FREEZE ////
352
class FreezeBase : public StackObj {
354
JavaThread* const _thread;
355
ContinuationWrapper& _cont;
356
bool _barriers; // only set when we allocate a chunk
357
const bool _preempt; // used only on the slow path
358
const intptr_t * const _frame_sp; // Top frame sp for this freeze
360
intptr_t* _bottom_address;
362
int _freeze_size; // total size of all frames plus metadata in words.
363
int _total_align_size;
365
intptr_t* _cont_stack_top;
366
intptr_t* _cont_stack_bottom;
368
CONT_JFR_ONLY(FreezeThawJfrInfo _jfr_info;)
371
intptr_t* _orig_chunk_sp;
372
int _fast_freeze_size;
376
JvmtiSampledObjectAllocEventCollector* _jvmti_event_collector;
378
NOT_PRODUCT(int _frames;)
379
DEBUG_ONLY(intptr_t* _last_write;)
381
inline FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp);
384
NOINLINE freeze_result freeze_slow();
385
void freeze_fast_existing_chunk();
387
CONT_JFR_ONLY(FreezeThawJfrInfo& jfr_info() { return _jfr_info; })
388
void set_jvmti_event_collector(JvmtiSampledObjectAllocEventCollector* jsoaec) { _jvmti_event_collector = jsoaec; }
390
inline int size_if_fast_freeze_available();
393
bool check_valid_fast_path();
397
inline void init_rest();
398
void throw_stack_overflow_on_humongous_chunk();
401
inline void copy_to_chunk(intptr_t* from, intptr_t* to, int size);
402
inline void unwind_frames();
403
inline void patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp);
406
virtual stackChunkOop allocate_chunk_slow(size_t stack_size, int argsize_md) = 0;
408
int cont_size() { return pointer_delta_as_int(_cont_stack_bottom, _cont_stack_top); }
412
frame freeze_start_frame();
413
frame freeze_start_frame_safepoint_stub(frame f);
414
NOINLINE freeze_result recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top);
415
inline frame freeze_start_frame_yield_stub(frame f);
416
template<typename FKind>
417
inline freeze_result recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize);
418
inline void before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame);
419
inline void after_freeze_java_frame(const frame& hf, bool is_bottom_frame);
420
freeze_result finalize_freeze(const frame& callee, frame& caller, int argsize);
421
void patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame);
422
NOINLINE freeze_result recurse_freeze_interpreted_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
423
freeze_result recurse_freeze_compiled_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted);
424
NOINLINE freeze_result recurse_freeze_stub_frame(frame& f, frame& caller);
425
NOINLINE void finish_freeze(const frame& f, const frame& top);
427
inline bool stack_overflow();
429
static frame sender(const frame& f) { return f.is_interpreted_frame() ? sender<ContinuationHelper::InterpretedFrame>(f)
430
: sender<ContinuationHelper::NonInterpretedUnknownFrame>(f); }
431
template<typename FKind> static inline frame sender(const frame& f);
432
template<typename FKind> frame new_heap_frame(frame& f, frame& caller);
433
inline void set_top_frame_metadata_pd(const frame& hf);
434
inline void patch_pd(frame& callee, const frame& caller);
435
void adjust_interpreted_frame_unextended_sp(frame& f);
436
static inline void relativize_interpreted_frame_metadata(const frame& f, const frame& hf);
439
void freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_JFR_ONLY(COMMA bool chunk_is_allocated));
440
bool freeze_fast_new_chunk(stackChunkOop chunk);
443
template <typename ConfigT>
444
class Freeze : public FreezeBase {
446
stackChunkOop allocate_chunk(size_t stack_size, int argsize_md);
449
inline Freeze(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp)
450
: FreezeBase(thread, cont, frame_sp) {}
452
freeze_result try_freeze_fast();
455
virtual stackChunkOop allocate_chunk_slow(size_t stack_size, int argsize_md) override { return allocate_chunk(stack_size, argsize_md); }
458
FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp) :
459
_thread(thread), _cont(cont), _barriers(false), _preempt(false), _frame_sp(frame_sp) {
460
DEBUG_ONLY(_jvmti_event_collector = nullptr;)
462
assert(_thread != nullptr, "");
463
assert(_thread->last_continuation()->entry_sp() == _cont.entrySP(), "");
465
DEBUG_ONLY(_cont.entry()->verify_cookie();)
467
assert(!Interpreter::contains(_cont.entryPC()), "");
469
_bottom_address = _cont.entrySP() - _cont.entry_frame_extension();
471
if (((intptr_t)_bottom_address & 0xf) != 0) {
474
assert(is_aligned(_bottom_address, frame::frame_alignment), "");
477
log_develop_trace(continuations)("bottom_address: " INTPTR_FORMAT " entrySP: " INTPTR_FORMAT " argsize: " PTR_FORMAT,
478
p2i(_bottom_address), p2i(_cont.entrySP()), (_cont.entrySP() - _bottom_address) << LogBytesPerWord);
479
assert(_bottom_address != nullptr, "");
480
assert(_bottom_address <= _cont.entrySP(), "");
481
DEBUG_ONLY(_last_write = nullptr;)
483
assert(_cont.chunk_invariant(), "");
484
assert(!Interpreter::contains(_cont.entryPC()), "");
485
#if !defined(PPC64) || defined(ZERO)
486
static const int doYield_stub_frame_size = frame::metadata_words;
488
static const int doYield_stub_frame_size = frame::native_abi_reg_args_size >> LogBytesPerWord;
490
assert(SharedRuntime::cont_doYield_stub()->frame_size() == doYield_stub_frame_size, "");
492
// properties of the continuation on the stack; all sizes are in words
493
_cont_stack_top = frame_sp + doYield_stub_frame_size; // we don't freeze the doYield stub frame
494
_cont_stack_bottom = _cont.entrySP() + (_cont.argsize() == 0 ? frame::metadata_words_at_top : 0)
495
- ContinuationHelper::frame_align_words(_cont.argsize()); // see alignment in thaw
497
log_develop_trace(continuations)("freeze size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT,
498
cont_size(), _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom));
499
assert(cont_size() > 0, "");
502
void FreezeBase::init_rest() { // we want to postpone some initialization after chunk handling
504
_total_align_size = 0;
505
NOT_PRODUCT(_frames = 0;)
508
void FreezeBase::copy_to_chunk(intptr_t* from, intptr_t* to, int size) {
509
stackChunkOop chunk = _cont.tail();
510
chunk->copy_from_stack_to_chunk(from, to, size);
511
CONT_JFR_ONLY(_jfr_info.record_size_copied(size);)
514
if (_last_write != nullptr) {
515
assert(_last_write == to + size, "Missed a spot: _last_write: " INTPTR_FORMAT " to+size: " INTPTR_FORMAT
516
" stack_size: %d _last_write offset: " PTR_FORMAT " to+size: " PTR_FORMAT, p2i(_last_write), p2i(to+size),
517
chunk->stack_size(), _last_write-chunk->start_address(), to+size-chunk->start_address());
523
// Called _after_ the last possible safepoint during the freeze operation (chunk allocation)
524
void FreezeBase::unwind_frames() {
525
ContinuationEntry* entry = _cont.entry();
526
entry->flush_stack_processing(_thread);
527
set_anchor_to_entry(_thread, entry);
530
template <typename ConfigT>
531
freeze_result Freeze<ConfigT>::try_freeze_fast() {
532
assert(_thread->thread_state() == _thread_in_vm, "");
533
assert(_thread->cont_fastpath(), "");
535
DEBUG_ONLY(_fast_freeze_size = size_if_fast_freeze_available();)
536
assert(_fast_freeze_size == 0, "");
538
stackChunkOop chunk = allocate_chunk(cont_size() + frame::metadata_words, _cont.argsize() + frame::metadata_words_at_top);
539
if (freeze_fast_new_chunk(chunk)) {
542
if (_thread->has_pending_exception()) {
543
return freeze_exception;
546
// TODO R REMOVE when deopt change is fixed
547
assert(!_thread->cont_fastpath() || _barriers, "");
548
log_develop_trace(continuations)("-- RETRYING SLOW --");
549
return freeze_slow();
552
// Returns size needed if the continuation fits, otherwise 0.
553
int FreezeBase::size_if_fast_freeze_available() {
554
stackChunkOop chunk = _cont.tail();
555
if (chunk == nullptr || chunk->is_gc_mode() || chunk->requires_barriers() || chunk->has_mixed_frames()) {
556
log_develop_trace(continuations)("chunk available %s", chunk == nullptr ? "no chunk" : "chunk requires barriers");
560
int total_size_needed = cont_size();
561
const int chunk_sp = chunk->sp();
563
// argsize can be nonzero if we have a caller, but the caller could be in a non-empty parent chunk,
564
// so we subtract it only if we overlap with the caller, i.e. the current chunk isn't empty.
565
// Consider leaving the chunk's argsize set when emptying it and removing the following branch,
566
// although that would require changing stackChunkOopDesc::is_empty
567
if (!chunk->is_empty()) {
568
total_size_needed -= _cont.argsize() + frame::metadata_words_at_top;
571
int chunk_free_room = chunk_sp - frame::metadata_words_at_bottom;
572
bool available = chunk_free_room >= total_size_needed;
573
log_develop_trace(continuations)("chunk available: %s size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT,
574
available ? "yes" : "no" , total_size_needed, _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom));
575
return available ? total_size_needed : 0;
578
void FreezeBase::freeze_fast_existing_chunk() {
579
stackChunkOop chunk = _cont.tail();
581
DEBUG_ONLY(_fast_freeze_size = size_if_fast_freeze_available();)
582
assert(_fast_freeze_size > 0, "");
584
if (!chunk->is_empty()) { // we are copying into a non-empty chunk
585
DEBUG_ONLY(_empty = false;)
586
DEBUG_ONLY(_orig_chunk_sp = chunk->sp_address();)
589
intptr_t* retaddr_slot = (chunk->sp_address()
590
- frame::sender_sp_ret_address_offset());
591
assert(ContinuationHelper::return_address_at(retaddr_slot) == chunk->pc(),
592
"unexpected saved return address");
596
// the chunk's sp before the freeze, adjusted to point beyond the stack-passed arguments in the topmost frame
597
// we overlap; we'll overwrite the chunk's top frame's callee arguments
598
const int chunk_start_sp = chunk->sp() + _cont.argsize() + frame::metadata_words_at_top;
599
assert(chunk_start_sp <= chunk->stack_size(), "sp not pointing into stack");
601
// increase max_size by what we're freezing minus the overlap
602
chunk->set_max_thawing_size(chunk->max_thawing_size() + cont_size() - _cont.argsize() - frame::metadata_words_at_top);
604
intptr_t* const bottom_sp = _cont_stack_bottom - _cont.argsize() - frame::metadata_words_at_top;
605
assert(bottom_sp == _bottom_address, "");
606
// Because the chunk isn't empty, we know there's a caller in the chunk, therefore the bottom-most frame
607
// should have a return barrier (installed back when we thawed it).
610
intptr_t* retaddr_slot = (bottom_sp
611
- frame::sender_sp_ret_address_offset());
612
assert(ContinuationHelper::return_address_at(retaddr_slot)
613
== StubRoutines::cont_returnBarrier(),
614
"should be the continuation return barrier");
617
// We copy the fp from the chunk back to the stack because it contains some caller data,
618
// including, possibly, an oop that might have gone stale since we thawed.
619
patch_stack_pd(bottom_sp, chunk->sp_address());
620
// we don't patch the return pc at this time, so as not to make the stack unwalkable for async walks
622
freeze_fast_copy(chunk, chunk_start_sp CONT_JFR_ONLY(COMMA false));
623
} else { // the chunk is empty
624
const int chunk_start_sp = chunk->stack_size();
626
DEBUG_ONLY(_empty = true;)
627
DEBUG_ONLY(_orig_chunk_sp = chunk->start_address() + chunk_start_sp;)
629
chunk->set_max_thawing_size(cont_size());
630
chunk->set_bottom(chunk_start_sp - _cont.argsize() - frame::metadata_words_at_top);
631
chunk->set_sp(chunk->bottom());
633
freeze_fast_copy(chunk, chunk_start_sp CONT_JFR_ONLY(COMMA false));
637
bool FreezeBase::freeze_fast_new_chunk(stackChunkOop chunk) {
638
DEBUG_ONLY(_empty = true;)
641
_cont.set_tail(chunk);
643
if (UNLIKELY(chunk == nullptr || !_thread->cont_fastpath() || _barriers)) { // OOME/probably humongous
644
log_develop_trace(continuations)("Retrying slow. Barriers: %d", _barriers);
648
chunk->set_max_thawing_size(cont_size());
650
// in a fresh chunk, we freeze *with* the bottom-most frame's stack arguments.
651
// They'll then be stored twice: in the chunk and in the parent chunk's top frame
652
const int chunk_start_sp = cont_size() + frame::metadata_words;
653
assert(chunk_start_sp == chunk->stack_size(), "");
655
DEBUG_ONLY(_orig_chunk_sp = chunk->start_address() + chunk_start_sp;)
657
freeze_fast_copy(chunk, chunk_start_sp CONT_JFR_ONLY(COMMA true));
662
void FreezeBase::freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_JFR_ONLY(COMMA bool chunk_is_allocated)) {
663
assert(chunk != nullptr, "");
664
assert(!chunk->has_mixed_frames(), "");
665
assert(!chunk->is_gc_mode(), "");
666
assert(!chunk->has_bitmap(), "");
667
assert(!chunk->requires_barriers(), "");
668
assert(chunk == _cont.tail(), "");
670
// We unwind frames after the last safepoint so that the GC will have found the oops in the frames, but before
671
// writing into the chunk. This is so that an asynchronous stack walk (not at a safepoint) that suspends us here
672
// will either see no continuation on the stack, or a consistent chunk.
675
log_develop_trace(continuations)("freeze_fast start: chunk " INTPTR_FORMAT " size: %d orig sp: %d argsize: %d",
676
p2i((oopDesc*)chunk), chunk->stack_size(), chunk_start_sp, _cont.argsize());
677
assert(chunk_start_sp <= chunk->stack_size(), "");
678
assert(chunk_start_sp >= cont_size(), "no room in the chunk");
680
const int chunk_new_sp = chunk_start_sp - cont_size(); // the chunk's new sp, after freeze
681
assert(!(_fast_freeze_size > 0) || _orig_chunk_sp - (chunk->start_address() + chunk_new_sp) == _fast_freeze_size, "");
683
intptr_t* chunk_top = chunk->start_address() + chunk_new_sp;
686
intptr_t* retaddr_slot = (_orig_chunk_sp
687
- frame::sender_sp_ret_address_offset());
688
assert(ContinuationHelper::return_address_at(retaddr_slot) == chunk->pc(),
689
"unexpected saved return address");
693
log_develop_trace(continuations)("freeze_fast start: " INTPTR_FORMAT " sp: %d chunk_top: " INTPTR_FORMAT,
694
p2i(chunk->start_address()), chunk_new_sp, p2i(chunk_top));
695
intptr_t* from = _cont_stack_top - frame::metadata_words_at_bottom;
696
intptr_t* to = chunk_top - frame::metadata_words_at_bottom;
697
copy_to_chunk(from, to, cont_size() + frame::metadata_words_at_bottom);
698
// Because we're not patched yet, the chunk is now in a bad state
700
// patch return pc of the bottom-most frozen frame (now in the chunk)
701
// with the actual caller's return address
702
intptr_t* chunk_bottom_retaddr_slot = (chunk_top + cont_size()
704
- frame::metadata_words_at_top
705
- frame::sender_sp_ret_address_offset());
708
assert(ContinuationHelper::return_address_at(chunk_bottom_retaddr_slot)
709
== StubRoutines::cont_returnBarrier(),
710
"should be the continuation return barrier");
713
ContinuationHelper::patch_return_address_at(chunk_bottom_retaddr_slot,
716
// We're always writing to a young chunk, so the GC can't see it until the next safepoint.
717
chunk->set_sp(chunk_new_sp);
718
// set chunk->pc to the return address of the topmost frame in the chunk
719
chunk->set_pc(ContinuationHelper::return_address_at(
720
_cont_stack_top - frame::sender_sp_ret_address_offset()));
724
log_develop_trace(continuations)("FREEZE CHUNK #" INTPTR_FORMAT " (young)", _cont.hash());
725
LogTarget(Trace, continuations) lt;
726
if (lt.develop_is_enabled()) {
728
chunk->print_on(true, &ls);
732
assert(_cont.chunk_invariant(), "");
736
EventContinuationFreezeFast e;
737
if (e.should_commit()) {
738
e.set_id(cast_from_oop<u8>(chunk));
739
DEBUG_ONLY(e.set_allocate(chunk_is_allocated);)
740
e.set_size(cont_size() << LogBytesPerWord);
746
NOINLINE freeze_result FreezeBase::freeze_slow() {
751
log_develop_trace(continuations)("freeze_slow #" INTPTR_FORMAT, _cont.hash());
752
assert(_thread->thread_state() == _thread_in_vm || _thread->thread_state() == _thread_blocked, "");
755
EventContinuationFreezeSlow e;
756
if (e.should_commit()) {
757
e.set_id(cast_from_oop<u8>(_cont.continuation()));
764
HandleMark hm(Thread::current());
766
frame f = freeze_start_frame();
768
LogTarget(Debug, continuations) lt;
769
if (lt.develop_is_enabled()) {
774
frame caller; // the frozen caller in the chunk
775
freeze_result res = recurse_freeze(f, caller, 0, false, true);
777
if (res == freeze_ok) {
778
finish_freeze(f, caller);
785
frame FreezeBase::freeze_start_frame() {
786
frame f = _thread->last_frame();
787
if (LIKELY(!_preempt)) {
788
return freeze_start_frame_yield_stub(f);
790
return freeze_start_frame_safepoint_stub(f);
794
frame FreezeBase::freeze_start_frame_yield_stub(frame f) {
795
assert(SharedRuntime::cont_doYield_stub()->contains(f.pc()), "must be");
796
f = sender<ContinuationHelper::NonInterpretedUnknownFrame>(f);
797
assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), "");
801
frame FreezeBase::freeze_start_frame_safepoint_stub(frame f) {
802
#if (defined(X86) || defined(AARCH64) || defined(RISCV64)) && !defined(ZERO)
803
f.set_fp(f.real_fp()); // f.set_fp(*Frame::callee_link_address(f)); // ????
807
if (!Interpreter::contains(f.pc())) {
808
assert(ContinuationHelper::Frame::is_stub(f.cb()), "must be");
809
assert(f.oop_map() != nullptr, "must be");
811
if (Interpreter::contains(ContinuationHelper::StubFrame::return_pc(f))) {
812
f = sender<ContinuationHelper::StubFrame>(f); // Safepoint stub in interpreter
815
assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), "");
819
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
820
NOINLINE freeze_result FreezeBase::recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top) {
821
assert(f.unextended_sp() < _bottom_address, ""); // see recurse_freeze_java_frame
822
assert(f.is_interpreted_frame() || ((top && _preempt) == ContinuationHelper::Frame::is_stub(f.cb())), "");
824
if (stack_overflow()) {
825
return freeze_exception;
828
if (f.is_compiled_frame()) {
829
if (UNLIKELY(f.oop_map() == nullptr)) {
830
// special native frame
831
return freeze_pinned_native;
833
return recurse_freeze_compiled_frame(f, caller, callee_argsize, callee_interpreted);
834
} else if (f.is_interpreted_frame()) {
835
assert((_preempt && top) || !f.interpreter_frame_method()->is_native(), "");
836
if (_preempt && top && f.interpreter_frame_method()->is_native()) {
838
return freeze_pinned_native;
841
return recurse_freeze_interpreted_frame(f, caller, callee_argsize, callee_interpreted);
842
} else if (_preempt && top && ContinuationHelper::Frame::is_stub(f.cb())) {
843
return recurse_freeze_stub_frame(f, caller);
845
return freeze_pinned_native;
849
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
850
// See also StackChunkFrameStream<frame_kind>::frame_size()
851
template<typename FKind>
852
inline freeze_result FreezeBase::recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize) {
853
assert(FKind::is_instance(f), "");
855
assert(fsize > 0, "");
856
assert(argsize >= 0, "");
857
_freeze_size += fsize;
858
NOT_PRODUCT(_frames++;)
860
assert(FKind::frame_bottom(f) <= _bottom_address, "");
862
// We don't use FKind::frame_bottom(f) == _bottom_address because on x64 there's sometimes an extra word between
863
// enterSpecial and an interpreted frame
864
if (FKind::frame_bottom(f) >= _bottom_address - 1) {
865
return finalize_freeze(f, caller, argsize); // recursion end
867
frame senderf = sender<FKind>(f);
868
assert(FKind::interpreted || senderf.sp() == senderf.unextended_sp(), "");
869
freeze_result result = recurse_freeze(senderf, caller, argsize, FKind::interpreted, false); // recursive call
874
inline void FreezeBase::before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame) {
875
LogTarget(Trace, continuations) lt;
876
if (lt.develop_is_enabled()) {
878
ls.print_cr("======== FREEZING FRAME interpreted: %d bottom: %d", f.is_interpreted_frame(), is_bottom_frame);
879
ls.print_cr("fsize: %d argsize: %d", fsize, argsize);
880
f.print_value_on(&ls, nullptr);
882
assert(caller.is_interpreted_frame() == Interpreter::contains(caller.pc()), "");
885
inline void FreezeBase::after_freeze_java_frame(const frame& hf, bool is_bottom_frame) {
886
LogTarget(Trace, continuations) lt;
887
if (lt.develop_is_enabled()) {
889
DEBUG_ONLY(hf.print_value_on(&ls, nullptr);)
890
assert(hf.is_heap_frame(), "should be");
891
DEBUG_ONLY(print_frame_layout(hf, false, &ls);)
892
if (is_bottom_frame) {
893
ls.print_cr("bottom h-frame:");
899
// The parameter argsize_md includes metadata that has to be part of caller/callee overlap.
900
// See also StackChunkFrameStream<frame_kind>::frame_size()
901
freeze_result FreezeBase::finalize_freeze(const frame& callee, frame& caller, int argsize_md) {
902
int argsize = argsize_md - frame::metadata_words_at_top;
903
assert(callee.is_interpreted_frame()
904
|| callee.cb()->as_nmethod()->is_osr_method()
905
|| argsize == _cont.argsize(), "argsize: %d cont.argsize: %d", argsize, _cont.argsize());
906
log_develop_trace(continuations)("bottom: " INTPTR_FORMAT " count %d size: %d argsize: %d",
907
p2i(_bottom_address), _frames, _freeze_size << LogBytesPerWord, argsize);
909
LogTarget(Trace, continuations) lt;
912
bool empty = _cont.is_empty();
913
log_develop_trace(continuations)("empty: %d", empty);
916
stackChunkOop chunk = _cont.tail();
918
assert(chunk == nullptr || (chunk->max_thawing_size() == 0) == chunk->is_empty(), "");
920
_freeze_size += frame::metadata_words; // for top frame's metadata
922
int overlap = 0; // the args overlap the caller -- if there is one in this chunk and is of the same kind
923
int unextended_sp = -1;
924
if (chunk != nullptr) {
925
if (!chunk->is_empty()) {
926
StackChunkFrameStream<ChunkFrames::Mixed> last(chunk);
927
unextended_sp = chunk->to_offset(StackChunkFrameStream<ChunkFrames::Mixed>(chunk).unextended_sp());
928
bool top_interpreted = Interpreter::contains(chunk->pc());
929
if (callee.is_interpreted_frame() == top_interpreted) {
930
overlap = argsize_md;
933
unextended_sp = chunk->stack_size() - frame::metadata_words_at_top;
937
log_develop_trace(continuations)("finalize _size: %d overlap: %d unextended_sp: %d", _freeze_size, overlap, unextended_sp);
939
_freeze_size -= overlap;
940
assert(_freeze_size >= 0, "");
942
assert(chunk == nullptr || chunk->is_empty()
943
|| unextended_sp == chunk->to_offset(StackChunkFrameStream<ChunkFrames::Mixed>(chunk).unextended_sp()), "");
944
assert(chunk != nullptr || unextended_sp < _freeze_size, "");
946
// _barriers can be set to true by an allocation in freeze_fast, in which case the chunk is available
947
bool allocated_old_in_freeze_fast = _barriers;
948
assert(!allocated_old_in_freeze_fast || (unextended_sp >= _freeze_size && chunk->is_empty()),
949
"Chunk allocated in freeze_fast is of insufficient size "
950
"unextended_sp: %d size: %d is_empty: %d", unextended_sp, _freeze_size, chunk->is_empty());
951
assert(!allocated_old_in_freeze_fast || (!UseZGC && !UseG1GC), "Unexpected allocation");
953
DEBUG_ONLY(bool empty_chunk = true);
954
if (unextended_sp < _freeze_size || chunk->is_gc_mode() || (!allocated_old_in_freeze_fast && chunk->requires_barriers())) {
955
// ALLOCATE NEW CHUNK
957
if (lt.develop_is_enabled()) {
959
if (chunk == nullptr) {
960
ls.print_cr("no chunk");
962
ls.print_cr("chunk barriers: %d _size: %d free size: %d",
963
chunk->requires_barriers(), _freeze_size, chunk->sp() - frame::metadata_words);
964
chunk->print_on(&ls);
968
_freeze_size += overlap; // we're allocating a new chunk, so no overlap
971
chunk = allocate_chunk_slow(_freeze_size, argsize_md);
972
if (chunk == nullptr) {
973
return freeze_exception;
977
_cont.set_tail(chunk);
978
assert(chunk->is_empty(), "");
980
// REUSE EXISTING CHUNK
981
log_develop_trace(continuations)("Reusing chunk mixed: %d empty: %d", chunk->has_mixed_frames(), chunk->is_empty());
982
if (chunk->is_empty()) {
983
int sp = chunk->stack_size() - argsize_md;
985
chunk->set_bottom(sp);
986
_freeze_size += overlap;
987
assert(chunk->max_thawing_size() == 0, "");
988
} DEBUG_ONLY(else empty_chunk = false;)
990
assert(!chunk->is_gc_mode(), "");
991
assert(!chunk->has_bitmap(), "");
992
chunk->set_has_mixed_frames(true);
994
assert(chunk->requires_barriers() == _barriers, "");
995
assert(!_barriers || chunk->is_empty(), "");
997
assert(!chunk->is_empty() || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).is_done(), "");
998
assert(!chunk->is_empty() || StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame().is_empty(), "");
1000
// We unwind frames after the last safepoint so that the GC will have found the oops in the frames, but before
1001
// writing into the chunk. This is so that an asynchronous stack walk (not at a safepoint) that suspends us here
1002
// will either see no continuation or a consistent chunk.
1005
chunk->set_max_thawing_size(chunk->max_thawing_size() + _freeze_size - frame::metadata_words);
1007
if (lt.develop_is_enabled()) {
1009
ls.print_cr("top chunk:");
1010
chunk->print_on(&ls);
1013
// The topmost existing frame in the chunk; or an empty frame if the chunk is empty
1014
caller = StackChunkFrameStream<ChunkFrames::Mixed>(chunk).to_frame();
1016
DEBUG_ONLY(_last_write = caller.unextended_sp() + (empty_chunk ? argsize_md : overlap);)
1018
assert(chunk->is_in_chunk(_last_write - _freeze_size),
1019
"last_write-size: " INTPTR_FORMAT " start: " INTPTR_FORMAT, p2i(_last_write-_freeze_size), p2i(chunk->start_address()));
1021
if (lt.develop_is_enabled()) {
1023
ls.print_cr("top hframe before (freeze):");
1024
assert(caller.is_heap_frame(), "should be");
1025
caller.print_on(&ls);
1028
assert(!empty || Continuation::is_continuation_entry_frame(callee, nullptr), "");
1030
frame entry = sender(callee);
1032
assert(Continuation::is_return_barrier_entry(entry.pc()) || Continuation::is_continuation_enterSpecial(entry), "");
1033
assert(callee.is_interpreted_frame() || entry.sp() == entry.unextended_sp(), "");
1036
return freeze_ok_bottom;
1039
void FreezeBase::patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame) {
1040
if (is_bottom_frame) {
1041
// If we're the bottom frame, we need to replace the return barrier with the real
1043
address last_pc = caller.pc();
1044
assert((last_pc == nullptr) == _cont.tail()->is_empty(), "");
1045
ContinuationHelper::Frame::patch_pc(caller, last_pc);
1047
assert(!caller.is_empty(), "");
1050
patch_pd(hf, caller);
1052
if (f.is_interpreted_frame()) {
1053
assert(hf.is_heap_frame(), "should be");
1054
ContinuationHelper::InterpretedFrame::patch_sender_sp(hf, caller);
1058
if (hf.is_compiled_frame()) {
1059
if (f.is_deoptimized_frame()) { // TODO DEOPT: long term solution: unroll on freeze and patch pc
1060
log_develop_trace(continuations)("Freezing deoptimized frame");
1061
assert(f.cb()->as_nmethod()->is_deopt_pc(f.raw_pc()), "");
1062
assert(f.cb()->as_nmethod()->is_deopt_pc(ContinuationHelper::Frame::real_pc(f)), "");
1069
static void verify_frame_top(const frame& f, intptr_t* top) {
1071
InterpreterOopMap mask;
1072
f.interpreted_frame_oop_map(&mask);
1073
assert(top <= ContinuationHelper::InterpretedFrame::frame_top(f, &mask),
1074
"frame_top: " INTPTR_FORMAT " Interpreted::frame_top: " INTPTR_FORMAT,
1075
p2i(top), p2i(ContinuationHelper::InterpretedFrame::frame_top(f, &mask)));
1079
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
1080
// See also StackChunkFrameStream<frame_kind>::frame_size()
1081
NOINLINE freeze_result FreezeBase::recurse_freeze_interpreted_frame(frame& f, frame& caller,
1082
int callee_argsize /* incl. metadata */,
1083
bool callee_interpreted) {
1084
adjust_interpreted_frame_unextended_sp(f);
1086
// The frame's top never includes the stack arguments to the callee
1087
intptr_t* const stack_frame_top = ContinuationHelper::InterpretedFrame::frame_top(f, callee_argsize, callee_interpreted);
1088
intptr_t* const stack_frame_bottom = ContinuationHelper::InterpretedFrame::frame_bottom(f);
1089
const int fsize = pointer_delta_as_int(stack_frame_bottom, stack_frame_top);
1091
DEBUG_ONLY(verify_frame_top(f, stack_frame_top));
1093
Method* frame_method = ContinuationHelper::Frame::frame_method(f);
1094
// including metadata between f and its args
1095
const int argsize = ContinuationHelper::InterpretedFrame::stack_argsize(f) + frame::metadata_words_at_top;
1097
log_develop_trace(continuations)("recurse_freeze_interpreted_frame %s _size: %d fsize: %d argsize: %d",
1098
frame_method->name_and_sig_as_C_string(), _freeze_size, fsize, argsize);
1099
// we'd rather not yield inside methods annotated with @JvmtiMountTransition
1100
assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), "");
1102
freeze_result result = recurse_freeze_java_frame<ContinuationHelper::InterpretedFrame>(f, caller, fsize, argsize);
1103
if (UNLIKELY(result > freeze_ok_bottom)) {
1107
bool is_bottom_frame = result == freeze_ok_bottom;
1108
assert(!caller.is_empty() || is_bottom_frame, "");
1110
DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, 0, is_bottom_frame);)
1112
frame hf = new_heap_frame<ContinuationHelper::InterpretedFrame>(f, caller);
1113
_total_align_size += frame::align_wiggle; // add alignment room for internal interpreted frame alignment on AArch64/PPC64
1115
intptr_t* heap_frame_top = ContinuationHelper::InterpretedFrame::frame_top(hf, callee_argsize, callee_interpreted);
1116
intptr_t* heap_frame_bottom = ContinuationHelper::InterpretedFrame::frame_bottom(hf);
1117
assert(heap_frame_bottom == heap_frame_top + fsize, "");
1119
// Some architectures (like AArch64/PPC64/RISC-V) add padding between the locals and the fixed_frame to keep the fp 16-byte-aligned.
1120
// On those architectures we freeze the padding in order to keep the same fp-relative offsets in the fixed_frame.
1121
copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
1122
assert(!is_bottom_frame || !caller.is_interpreted_frame() || (heap_frame_top + fsize) == (caller.unextended_sp() + argsize), "");
1124
relativize_interpreted_frame_metadata(f, hf);
1126
patch(f, hf, caller, is_bottom_frame);
1128
CONT_JFR_ONLY(_jfr_info.record_interpreted_frame();)
1129
DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);)
1132
// Mark frame_method's GC epoch for class redefinition on_stack calculation.
1133
frame_method->record_gc_epoch();
1138
// The parameter callee_argsize includes metadata that has to be part of caller/callee overlap.
1139
// See also StackChunkFrameStream<frame_kind>::frame_size()
1140
freeze_result FreezeBase::recurse_freeze_compiled_frame(frame& f, frame& caller,
1141
int callee_argsize /* incl. metadata */,
1142
bool callee_interpreted) {
1143
// The frame's top never includes the stack arguments to the callee
1144
intptr_t* const stack_frame_top = ContinuationHelper::CompiledFrame::frame_top(f, callee_argsize, callee_interpreted);
1145
intptr_t* const stack_frame_bottom = ContinuationHelper::CompiledFrame::frame_bottom(f);
1146
// including metadata between f and its stackargs
1147
const int argsize = ContinuationHelper::CompiledFrame::stack_argsize(f) + frame::metadata_words_at_top;
1148
const int fsize = pointer_delta_as_int(stack_frame_bottom + argsize, stack_frame_top);
1150
log_develop_trace(continuations)("recurse_freeze_compiled_frame %s _size: %d fsize: %d argsize: %d",
1151
ContinuationHelper::Frame::frame_method(f) != nullptr ?
1152
ContinuationHelper::Frame::frame_method(f)->name_and_sig_as_C_string() : "",
1153
_freeze_size, fsize, argsize);
1154
// we'd rather not yield inside methods annotated with @JvmtiMountTransition
1155
assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), "");
1157
freeze_result result = recurse_freeze_java_frame<ContinuationHelper::CompiledFrame>(f, caller, fsize, argsize);
1158
if (UNLIKELY(result > freeze_ok_bottom)) {
1162
bool is_bottom_frame = result == freeze_ok_bottom;
1163
assert(!caller.is_empty() || is_bottom_frame, "");
1165
DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, argsize, is_bottom_frame);)
1167
frame hf = new_heap_frame<ContinuationHelper::CompiledFrame>(f, caller);
1169
intptr_t* heap_frame_top = ContinuationHelper::CompiledFrame::frame_top(hf, callee_argsize, callee_interpreted);
1171
copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
1172
assert(!is_bottom_frame || !caller.is_compiled_frame() || (heap_frame_top + fsize) == (caller.unextended_sp() + argsize), "");
1174
if (caller.is_interpreted_frame()) {
1175
_total_align_size += frame::align_wiggle; // See Thaw::align
1178
patch(f, hf, caller, is_bottom_frame);
1180
assert(is_bottom_frame || Interpreter::contains(ContinuationHelper::CompiledFrame::real_pc(caller)) == caller.is_interpreted_frame(), "");
1182
DEBUG_ONLY(after_freeze_java_frame(hf, is_bottom_frame);)
1187
NOINLINE freeze_result FreezeBase::recurse_freeze_stub_frame(frame& f, frame& caller) {
1188
intptr_t* const stack_frame_top = ContinuationHelper::StubFrame::frame_top(f, 0, 0);
1189
const int fsize = f.cb()->frame_size();
1191
log_develop_trace(continuations)("recurse_freeze_stub_frame %s _size: %d fsize: %d :: " INTPTR_FORMAT " - " INTPTR_FORMAT,
1192
f.cb()->name(), _freeze_size, fsize, p2i(stack_frame_top), p2i(stack_frame_top+fsize));
1194
// recurse_freeze_java_frame and freeze inlined here because we need to use a full RegisterMap for lock ownership
1195
NOT_PRODUCT(_frames++;)
1196
_freeze_size += fsize;
1198
RegisterMap map(_cont.thread(),
1199
RegisterMap::UpdateMap::include,
1200
RegisterMap::ProcessFrames::skip,
1201
RegisterMap::WalkContinuation::skip);
1202
map.set_include_argument_oops(false);
1203
ContinuationHelper::update_register_map<ContinuationHelper::StubFrame>(f, &map);
1204
f.oop_map()->update_register_map(&f, &map); // we have callee-save registers in this case
1205
frame senderf = sender<ContinuationHelper::StubFrame>(f);
1206
assert(senderf.unextended_sp() < _bottom_address - 1, "");
1207
assert(senderf.is_compiled_frame(), "");
1209
if (UNLIKELY(senderf.oop_map() == nullptr)) {
1211
return freeze_pinned_native;
1214
freeze_result result = recurse_freeze_compiled_frame(senderf, caller, 0, 0); // This might be deoptimized
1215
if (UNLIKELY(result > freeze_ok_bottom)) {
1218
assert(result != freeze_ok_bottom, "");
1219
assert(!caller.is_interpreted_frame(), "");
1221
DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, 0, false);)
1222
frame hf = new_heap_frame<ContinuationHelper::StubFrame>(f, caller);
1223
intptr_t* heap_frame_top = ContinuationHelper::StubFrame::frame_top(hf, 0, 0);
1224
copy_to_chunk(stack_frame_top, heap_frame_top, fsize);
1225
DEBUG_ONLY(after_freeze_java_frame(hf, false);)
1231
NOINLINE void FreezeBase::finish_freeze(const frame& f, const frame& top) {
1232
stackChunkOop chunk = _cont.tail();
1234
LogTarget(Trace, continuations) lt;
1235
if (lt.develop_is_enabled()) {
1237
assert(top.is_heap_frame(), "should be");
1241
set_top_frame_metadata_pd(top);
1243
chunk->set_sp(chunk->to_offset(top.sp()));
1244
chunk->set_pc(top.pc());
1246
chunk->set_max_thawing_size(chunk->max_thawing_size() + _total_align_size);
1248
// At this point the chunk is consistent
1250
if (UNLIKELY(_barriers)) {
1251
log_develop_trace(continuations)("do barriers on old chunk");
1252
// Serial and Parallel GC can allocate objects directly into the old generation.
1253
// Then we want to relativize the derived pointers eagerly so that
1254
// old chunks are all in GC mode.
1255
assert(!UseG1GC, "G1 can not deal with allocating outside of eden");
1256
assert(!UseZGC, "ZGC can not deal with allocating chunks visible to marking");
1257
if (UseShenandoahGC) {
1258
_cont.tail()->relativize_derived_pointers_concurrently();
1260
ContinuationGCSupport::transform_stack_chunk(_cont.tail());
1262
// For objects in the old generation we must maintain the remembered set
1263
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>();
1266
log_develop_trace(continuations)("finish_freeze: has_mixed_frames: %d", chunk->has_mixed_frames());
1267
if (lt.develop_is_enabled()) {
1269
chunk->print_on(true, &ls);
1272
if (lt.develop_is_enabled()) {
1274
ls.print_cr("top hframe after (freeze):");
1275
assert(_cont.last_frame().is_heap_frame(), "should be");
1276
_cont.last_frame().print_on(&ls);
1279
assert(_cont.chunk_invariant(), "");
1282
inline bool FreezeBase::stack_overflow() { // detect stack overflow in recursive native code
1283
JavaThread* t = !_preempt ? _thread : JavaThread::current();
1284
assert(t == JavaThread::current(), "");
1285
if (os::current_stack_pointer() < t->stack_overflow_state()->shadow_zone_safe_limit()) {
1287
ContinuationWrapper::SafepointOp so(t, _cont); // could also call _cont.done() instead
1288
Exceptions::_throw_msg(t, __FILE__, __LINE__, vmSymbols::java_lang_StackOverflowError(), "Stack overflow while freezing");
1295
class StackChunkAllocator : public MemAllocator {
1296
const size_t _stack_size;
1298
ContinuationWrapper& _continuation_wrapper;
1299
JvmtiSampledObjectAllocEventCollector* const _jvmti_event_collector;
1300
mutable bool _took_slow_path;
1302
// Does the minimal amount of initialization needed for a TLAB allocation.
1303
// We don't need to do a full initialization, as such an allocation need not be immediately walkable.
1304
virtual oop initialize(HeapWord* mem) const override {
1305
assert(_stack_size > 0, "");
1306
assert(_stack_size <= max_jint, "");
1307
assert(_word_size > _stack_size, "");
1309
// zero out fields (but not the stack)
1310
const size_t hs = oopDesc::header_size();
1311
Copy::fill_to_aligned_words(mem + hs, vmClasses::StackChunk_klass()->size_helper() - hs);
1313
int bottom = (int)_stack_size - _argsize_md;
1315
jdk_internal_vm_StackChunk::set_size(mem, (int)_stack_size);
1316
jdk_internal_vm_StackChunk::set_bottom(mem, bottom);
1317
jdk_internal_vm_StackChunk::set_sp(mem, bottom);
1322
stackChunkOop allocate_fast() const {
1327
HeapWord* const mem = MemAllocator::mem_allocate_inside_tlab_fast();
1328
if (mem == nullptr) {
1332
oop obj = initialize(mem);
1333
return stackChunkOopDesc::cast(obj);
1337
StackChunkAllocator(Klass* klass,
1342
ContinuationWrapper& continuation_wrapper,
1343
JvmtiSampledObjectAllocEventCollector* jvmti_event_collector)
1344
: MemAllocator(klass, word_size, thread),
1345
_stack_size(stack_size),
1346
_argsize_md(argsize_md),
1347
_continuation_wrapper(continuation_wrapper),
1348
_jvmti_event_collector(jvmti_event_collector),
1349
_took_slow_path(false) {}
1351
// Provides it's own, specialized allocation which skips instrumentation
1352
// if the memory can be allocated without going to a slow-path.
1353
stackChunkOop allocate() const {
1354
// First try to allocate without any slow-paths or instrumentation.
1355
stackChunkOop obj = allocate_fast();
1356
if (obj != nullptr) {
1360
// Now try full-blown allocation with all expensive operations,
1361
// including potentially safepoint operations.
1362
_took_slow_path = true;
1364
// Protect unhandled Loom oops
1365
ContinuationWrapper::SafepointOp so(_thread, _continuation_wrapper);
1368
_jvmti_event_collector->start();
1371
return stackChunkOopDesc::cast(MemAllocator::allocate());
1374
bool took_slow_path() const {
1375
return _took_slow_path;
1379
template <typename ConfigT>
1380
stackChunkOop Freeze<ConfigT>::allocate_chunk(size_t stack_size, int argsize_md) {
1381
log_develop_trace(continuations)("allocate_chunk allocating new chunk");
1383
InstanceStackChunkKlass* klass = InstanceStackChunkKlass::cast(vmClasses::StackChunk_klass());
1384
size_t size_in_words = klass->instance_size(stack_size);
1386
if (CollectedHeap::stack_chunk_max_size() > 0 && size_in_words >= CollectedHeap::stack_chunk_max_size()) {
1388
throw_stack_overflow_on_humongous_chunk();
1393
JavaThread* current = _preempt ? JavaThread::current() : _thread;
1394
assert(current == JavaThread::current(), "should be current");
1396
// Allocate the chunk.
1398
// This might safepoint while allocating, but all safepointing due to
1399
// instrumentation have been deferred. This property is important for
1400
// some GCs, as this ensures that the allocated object is in the young
1401
// generation / newly allocated memory.
1402
StackChunkAllocator allocator(klass, size_in_words, current, stack_size, argsize_md, _cont, _jvmti_event_collector);
1403
stackChunkOop chunk = allocator.allocate();
1405
if (chunk == nullptr) {
1406
return nullptr; // OOME
1409
// assert that chunk is properly initialized
1410
assert(chunk->stack_size() == (int)stack_size, "");
1411
assert(chunk->size() >= stack_size, "chunk->size(): %zu size: %zu", chunk->size(), stack_size);
1412
assert(chunk->sp() == chunk->bottom(), "");
1413
assert((intptr_t)chunk->start_address() % 8 == 0, "");
1414
assert(chunk->max_thawing_size() == 0, "");
1415
assert(chunk->pc() == nullptr, "");
1416
assert(chunk->is_empty(), "");
1417
assert(chunk->flags() == 0, "");
1418
assert(chunk->is_gc_mode() == false, "");
1420
// fields are uninitialized
1421
chunk->set_parent_access<IS_DEST_UNINITIALIZED>(_cont.last_nonempty_chunk());
1422
chunk->set_cont_access<IS_DEST_UNINITIALIZED>(_cont.continuation());
1426
if (ZGenerational) {
1427
ZStackChunkGCData::initialize(chunk);
1429
assert(!chunk->requires_barriers(), "ZGC always allocates in the young generation");
1433
#if INCLUDE_SHENANDOAHGC
1434
if (UseShenandoahGC) {
1435
_barriers = chunk->requires_barriers();
1439
if (!allocator.took_slow_path()) {
1440
// Guaranteed to be in young gen / newly allocated memory
1441
assert(!chunk->requires_barriers(), "Unfamiliar GC requires barriers on TLAB allocation");
1444
// Some GCs could put direct allocations in old gen for slow-path
1445
// allocations; need to explicitly check if that was the case.
1446
_barriers = chunk->requires_barriers();
1451
log_develop_trace(continuations)("allocation requires barriers");
1454
assert(chunk->parent() == nullptr || chunk->parent()->is_stackChunk(), "");
1459
void FreezeBase::throw_stack_overflow_on_humongous_chunk() {
1460
ContinuationWrapper::SafepointOp so(_thread, _cont); // could also call _cont.done() instead
1461
Exceptions::_throw_msg(_thread, __FILE__, __LINE__, vmSymbols::java_lang_StackOverflowError(), "Humongous stack chunk");
1465
static int num_java_frames(ContinuationWrapper& cont) {
1466
ResourceMark rm; // used for scope traversal in num_java_frames(nmethod*, address)
1468
for (stackChunkOop chunk = cont.tail(); chunk != nullptr; chunk = chunk->parent()) {
1469
count += chunk->num_java_frames();
1474
static void invalidate_jvmti_stack(JavaThread* thread) {
1475
if (thread->is_interp_only_mode()) {
1476
JvmtiThreadState *state = thread->jvmti_thread_state();
1477
if (state != nullptr)
1478
state->invalidate_cur_stack_depth();
1482
static void jvmti_yield_cleanup(JavaThread* thread, ContinuationWrapper& cont) {
1483
if (JvmtiExport::can_post_frame_pop()) {
1484
int num_frames = num_java_frames(cont);
1486
ContinuationWrapper::SafepointOp so(Thread::current(), cont);
1487
JvmtiExport::continuation_yield_cleanup(JavaThread::current(), num_frames);
1489
invalidate_jvmti_stack(thread);
1491
#endif // INCLUDE_JVMTI
1494
static bool monitors_on_stack(JavaThread* thread) {
1495
ContinuationEntry* ce = thread->last_continuation();
1496
RegisterMap map(thread,
1497
RegisterMap::UpdateMap::include,
1498
RegisterMap::ProcessFrames::include,
1499
RegisterMap::WalkContinuation::skip);
1500
map.set_include_argument_oops(false);
1501
for (frame f = thread->last_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) {
1502
if ((f.is_interpreted_frame() && ContinuationHelper::InterpretedFrame::is_owning_locks(f)) ||
1503
(f.is_compiled_frame() && ContinuationHelper::CompiledFrame::is_owning_locks(map.thread(), &map, f))) {
1510
// There are no interpreted frames if we're not called from the interpreter and we haven't ancountered an i2c
1511
// adapter or called Deoptimization::unpack_frames. As for native frames, upcalls from JNI also go through the
1512
// interpreter (see JavaCalls::call_helper), while the UpcallLinker explicitly sets cont_fastpath.
1513
bool FreezeBase::check_valid_fast_path() {
1514
ContinuationEntry* ce = _thread->last_continuation();
1515
RegisterMap map(_thread,
1516
RegisterMap::UpdateMap::skip,
1517
RegisterMap::ProcessFrames::skip,
1518
RegisterMap::WalkContinuation::skip);
1519
map.set_include_argument_oops(false);
1520
for (frame f = freeze_start_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) {
1521
if (!f.is_compiled_frame() || f.is_deoptimized_frame()) {
1529
static inline int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont) {
1530
verify_continuation(cont.continuation());
1531
assert(!cont.is_empty(), "");
1532
// This is done for the sake of the enterSpecial frame
1533
StackWatermarkSet::after_unwind(thread);
1535
log_develop_debug(continuations)("=== End of freeze cont ### #" INTPTR_FORMAT, cont.hash());
1540
static int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont, freeze_result res) {
1541
if (UNLIKELY(res != freeze_ok)) {
1542
verify_continuation(cont.continuation());
1543
log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
1547
JVMTI_ONLY(jvmti_yield_cleanup(thread, cont)); // can safepoint
1548
return freeze_epilog(thread, cont);
1551
template<typename ConfigT>
1552
static inline int freeze_internal(JavaThread* current, intptr_t* const sp) {
1553
assert(!current->has_pending_exception(), "");
1556
log_trace(continuations)("~~~~ freeze sp: " INTPTR_FORMAT, p2i(current->last_continuation()->entry_sp()));
1557
log_frames(current);
1560
CONT_JFR_ONLY(EventContinuationFreeze event;)
1562
ContinuationEntry* entry = current->last_continuation();
1564
oop oopCont = entry->cont_oop(current);
1565
assert(oopCont == current->last_continuation()->cont_oop(current), "");
1566
assert(ContinuationEntry::assert_entry_frame_laid_out(current), "");
1568
verify_continuation(oopCont);
1569
ContinuationWrapper cont(current, oopCont);
1570
log_develop_debug(continuations)("FREEZE #" INTPTR_FORMAT " " INTPTR_FORMAT, cont.hash(), p2i((oopDesc*)oopCont));
1572
assert(entry->is_virtual_thread() == (entry->scope(current) == java_lang_VirtualThread::vthread_scope()), "");
1574
assert(monitors_on_stack(current) == ((current->held_monitor_count() - current->jni_monitor_count()) > 0),
1575
"Held monitor count and locks on stack invariant: " INT64_FORMAT " JNI: " INT64_FORMAT, (int64_t)current->held_monitor_count(), (int64_t)current->jni_monitor_count());
1577
if (entry->is_pinned() || current->held_monitor_count() > 0) {
1578
log_develop_debug(continuations)("PINNED due to critical section/hold monitor");
1579
verify_continuation(cont.continuation());
1580
freeze_result res = entry->is_pinned() ? freeze_pinned_cs : freeze_pinned_monitor;
1581
log_develop_trace(continuations)("=== end of freeze (fail %d)", res);
1582
// Avoid Thread.yield() loops without safepoint polls.
1583
if (SafepointMechanism::should_process(current)) {
1584
cont.done(); // allow safepoint
1585
ThreadInVMfromJava tivmfj(current);
1590
Freeze<ConfigT> freeze(current, cont, sp);
1592
assert(!current->cont_fastpath() || freeze.check_valid_fast_path(), "");
1593
bool fast = UseContinuationFastPath && current->cont_fastpath();
1594
if (fast && freeze.size_if_fast_freeze_available() > 0) {
1595
freeze.freeze_fast_existing_chunk();
1596
CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);)
1597
freeze_epilog(current, cont);
1601
log_develop_trace(continuations)("chunk unavailable; transitioning to VM");
1602
assert(current == JavaThread::current(), "must be current thread except for preempt");
1604
// delays a possible JvmtiSampledObjectAllocEventCollector in alloc_chunk
1605
JvmtiSampledObjectAllocEventCollector jsoaec(false);
1606
freeze.set_jvmti_event_collector(&jsoaec);
1608
freeze_result res = fast ? freeze.try_freeze_fast() : freeze.freeze_slow();
1610
CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);)
1611
freeze_epilog(current, cont, res);
1612
cont.done(); // allow safepoint in the transition back to Java
1617
static freeze_result is_pinned0(JavaThread* thread, oop cont_scope, bool safepoint) {
1618
ContinuationEntry* entry = thread->last_continuation();
1619
if (entry == nullptr) {
1622
if (entry->is_pinned()) {
1623
return freeze_pinned_cs;
1624
} else if (thread->held_monitor_count() > 0) {
1625
return freeze_pinned_monitor;
1628
RegisterMap map(thread,
1629
RegisterMap::UpdateMap::include,
1630
RegisterMap::ProcessFrames::skip,
1631
RegisterMap::WalkContinuation::skip);
1632
map.set_include_argument_oops(false);
1633
frame f = thread->last_frame();
1636
f = f.sender(&map); // this is the yield frame
1637
} else { // safepoint yield
1638
#if (defined(X86) || defined(AARCH64) || defined(RISCV64)) && !defined(ZERO)
1639
f.set_fp(f.real_fp()); // Instead of this, maybe in ContinuationWrapper::set_last_frame always use the real_fp?
1643
if (!Interpreter::contains(f.pc())) {
1644
assert(ContinuationHelper::Frame::is_stub(f.cb()), "must be");
1645
assert(f.oop_map() != nullptr, "must be");
1646
f.oop_map()->update_register_map(&f, &map); // we have callee-save registers in this case
1651
if ((f.is_interpreted_frame() && f.interpreter_frame_method()->is_native()) || f.is_native_frame()) {
1652
return freeze_pinned_native;
1656
if (!Continuation::is_frame_in_continuation(entry, f)) {
1657
oop scope = jdk_internal_vm_Continuation::scope(entry->cont_oop(thread));
1658
if (scope == cont_scope) {
1661
intx monitor_count = entry->parent_held_monitor_count();
1662
entry = entry->parent();
1663
if (entry == nullptr) {
1666
if (entry->is_pinned()) {
1667
return freeze_pinned_cs;
1668
} else if (monitor_count > 0) {
1669
return freeze_pinned_monitor;
1676
/////////////// THAW ////
1678
static int thaw_size(stackChunkOop chunk) {
1679
int size = chunk->max_thawing_size();
1680
size += frame::metadata_words; // For the top pc+fp in push_return_frame or top = stack_sp - frame::metadata_words in thaw_fast
1681
size += 2*frame::align_wiggle; // in case of alignments at the top and bottom
1685
// make room on the stack for thaw
1686
// returns the size in bytes, or 0 on failure
1687
static inline int prepare_thaw_internal(JavaThread* thread, bool return_barrier) {
1688
log_develop_trace(continuations)("~~~~ prepare_thaw return_barrier: %d", return_barrier);
1690
assert(thread == JavaThread::current(), "");
1692
ContinuationEntry* ce = thread->last_continuation();
1693
assert(ce != nullptr, "");
1694
oop continuation = ce->cont_oop(thread);
1695
assert(continuation == get_continuation(thread), "");
1696
verify_continuation(continuation);
1698
stackChunkOop chunk = jdk_internal_vm_Continuation::tail(continuation);
1699
assert(chunk != nullptr, "");
1701
// The tail can be empty because it might still be available for another freeze.
1702
// However, here we want to thaw, so we get rid of it (it will be GCed).
1703
if (UNLIKELY(chunk->is_empty())) {
1704
chunk = chunk->parent();
1705
assert(chunk != nullptr, "");
1706
assert(!chunk->is_empty(), "");
1707
jdk_internal_vm_Continuation::set_tail(continuation, chunk);
1712
assert(chunk->max_thawing_size() > 0, "chunk invariant violated; expected to not be empty");
1714
// Only make space for the last chunk because we only thaw from the last chunk
1715
int size = thaw_size(chunk) << LogBytesPerWord;
1717
const address bottom = (address)thread->last_continuation()->entry_sp();
1718
// 300 is an estimate for stack size taken for this native code, in addition to StackShadowPages
1719
// for the Java frames in the check below.
1720
if (!stack_overflow_check(thread, size + 300, bottom)) {
1724
log_develop_trace(continuations)("prepare_thaw bottom: " INTPTR_FORMAT " top: " INTPTR_FORMAT " size: %d",
1725
p2i(bottom), p2i(bottom - size), size);
1729
class ThawBase : public StackObj {
1731
JavaThread* _thread;
1732
ContinuationWrapper& _cont;
1733
CONT_JFR_ONLY(FreezeThawJfrInfo _jfr_info;)
1735
intptr_t* _fastpath;
1737
intptr_t* _top_unextended_sp_before_thaw;
1739
DEBUG_ONLY(intptr_t* _top_stack_address);
1741
StackChunkFrameStream<ChunkFrames::Mixed> _stream;
1743
NOT_PRODUCT(int _frames;)
1746
ThawBase(JavaThread* thread, ContinuationWrapper& cont) :
1747
_thread(thread), _cont(cont),
1748
_fastpath(nullptr) {
1749
DEBUG_ONLY(_top_unextended_sp_before_thaw = nullptr;)
1750
assert (cont.tail() != nullptr, "no last chunk");
1751
DEBUG_ONLY(_top_stack_address = _cont.entrySP() - thaw_size(cont.tail());)
1754
void clear_chunk(stackChunkOop chunk);
1755
int remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize);
1756
void copy_from_chunk(intptr_t* from, intptr_t* to, int size);
1759
inline void prefetch_chunk_pd(void* start, int size_words);
1760
void patch_return(intptr_t* sp, bool is_last);
1763
NOINLINE intptr_t* thaw_slow(stackChunkOop chunk, bool return_barrier);
1766
void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top);
1767
template<typename FKind> bool recurse_thaw_java_frame(frame& caller, int num_frames);
1768
void finalize_thaw(frame& entry, int argsize);
1770
inline bool seen_by_gc();
1772
inline void before_thaw_java_frame(const frame& hf, const frame& caller, bool bottom, int num_frame);
1773
inline void after_thaw_java_frame(const frame& f, bool bottom);
1774
inline void patch(frame& f, const frame& caller, bool bottom);
1775
void clear_bitmap_bits(address start, address end);
1777
NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames);
1778
void recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller);
1779
void recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames);
1780
void finish_thaw(frame& f);
1782
void push_return_frame(frame& f);
1783
inline frame new_entry_frame();
1784
template<typename FKind> frame new_stack_frame(const frame& hf, frame& caller, bool bottom);
1785
inline void patch_pd(frame& f, const frame& sender);
1786
inline intptr_t* align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom);
1788
void maybe_set_fastpath(intptr_t* sp) { if (sp > _fastpath) _fastpath = sp; }
1790
static inline void derelativize_interpreted_frame_metadata(const frame& hf, const frame& f);
1793
CONT_JFR_ONLY(FreezeThawJfrInfo& jfr_info() { return _jfr_info; })
1796
template <typename ConfigT>
1797
class Thaw : public ThawBase {
1799
Thaw(JavaThread* thread, ContinuationWrapper& cont) : ThawBase(thread, cont) {}
1801
inline bool can_thaw_fast(stackChunkOop chunk) {
1803
&& _thread->cont_fastpath_thread_state()
1804
&& !chunk->has_thaw_slowpath_condition()
1805
&& !PreserveFramePointer;
1808
inline intptr_t* thaw(Continuation::thaw_kind kind);
1809
NOINLINE intptr_t* thaw_fast(stackChunkOop chunk);
1810
inline void patch_caller_links(intptr_t* sp, intptr_t* bottom);
1813
template <typename ConfigT>
1814
inline intptr_t* Thaw<ConfigT>::thaw(Continuation::thaw_kind kind) {
1815
verify_continuation(_cont.continuation());
1816
assert(!jdk_internal_vm_Continuation::done(_cont.continuation()), "");
1817
assert(!_cont.is_empty(), "");
1819
stackChunkOop chunk = _cont.tail();
1820
assert(chunk != nullptr, "guaranteed by prepare_thaw");
1821
assert(!chunk->is_empty(), "guaranteed by prepare_thaw");
1823
_barriers = chunk->requires_barriers();
1824
return (LIKELY(can_thaw_fast(chunk))) ? thaw_fast(chunk)
1825
: thaw_slow(chunk, kind != Continuation::thaw_top);
1828
class ReconstructedStack : public StackObj {
1829
intptr_t* _base; // _cont.entrySP(); // top of the entry frame
1833
ReconstructedStack(intptr_t* base, int thaw_size, int argsize)
1834
: _base(base), _thaw_size(thaw_size - (argsize == 0 ? frame::metadata_words_at_top : 0)), _argsize(argsize) {
1835
// The only possible source of misalignment is stack-passed arguments b/c compiled frames are 16-byte aligned.
1836
assert(argsize != 0 || (_base - _thaw_size) == ContinuationHelper::frame_align_pointer(_base - _thaw_size), "");
1837
// We're at most one alignment word away from entrySP
1838
assert(_base - 1 <= top() + total_size() + frame::metadata_words_at_bottom, "missed entry frame");
1841
int entry_frame_extension() const { return _argsize + (_argsize > 0 ? frame::metadata_words_at_top : 0); }
1843
// top and bottom stack pointers
1844
intptr_t* sp() const { return ContinuationHelper::frame_align_pointer(_base - _thaw_size); }
1845
intptr_t* bottom_sp() const { return ContinuationHelper::frame_align_pointer(_base - entry_frame_extension()); }
1847
// several operations operate on the totality of the stack being reconstructed,
1848
// including the metadata words
1849
intptr_t* top() const { return sp() - frame::metadata_words_at_bottom; }
1850
int total_size() const { return _thaw_size + frame::metadata_words_at_bottom; }
1853
inline void ThawBase::clear_chunk(stackChunkOop chunk) {
1854
chunk->set_sp(chunk->bottom());
1855
chunk->set_max_thawing_size(0);
1858
int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize) {
1860
StackChunkFrameStream<ChunkFrames::CompiledOnly> f(chunk);
1861
DEBUG_ONLY(intptr_t* const chunk_sp = chunk->start_address() + chunk->sp();)
1862
assert(chunk_sp == f.sp(), "");
1863
assert(chunk_sp == f.unextended_sp(), "");
1865
const int frame_size = f.cb()->frame_size();
1866
argsize = f.stack_argsize();
1868
f.next(SmallRegisterMap::instance, true /* stop */);
1869
empty = f.is_done();
1870
assert(!empty || argsize == chunk->argsize(), "");
1875
chunk->set_sp(chunk->sp() + frame_size);
1876
chunk->set_max_thawing_size(chunk->max_thawing_size() - frame_size);
1877
// We set chunk->pc to the return pc into the next frame
1878
chunk->set_pc(f.pc());
1881
intptr_t* retaddr_slot = (chunk_sp
1883
- frame::sender_sp_ret_address_offset());
1884
assert(f.pc() == ContinuationHelper::return_address_at(retaddr_slot),
1889
assert(empty == chunk->is_empty(), "");
1890
// returns the size required to store the frame on stack, and because it is a
1891
// compiled frame, it must include a copy of the arguments passed by the caller
1892
return frame_size + argsize + frame::metadata_words_at_top;
1895
void ThawBase::copy_from_chunk(intptr_t* from, intptr_t* to, int size) {
1896
assert(to >= _top_stack_address, "overwrote past thawing space"
1897
" to: " INTPTR_FORMAT " top_address: " INTPTR_FORMAT, p2i(to), p2i(_top_stack_address));
1898
assert(to + size <= _cont.entrySP(), "overwrote past thawing space");
1899
_cont.tail()->copy_from_chunk_to_stack(from, to, size);
1900
CONT_JFR_ONLY(_jfr_info.record_size_copied(size);)
1903
void ThawBase::patch_return(intptr_t* sp, bool is_last) {
1904
log_develop_trace(continuations)("thaw_fast patching -- sp: " INTPTR_FORMAT, p2i(sp));
1906
address pc = !is_last ? StubRoutines::cont_returnBarrier() : _cont.entryPC();
1907
ContinuationHelper::patch_return_address_at(
1908
sp - frame::sender_sp_ret_address_offset(),
1912
template <typename ConfigT>
1913
NOINLINE intptr_t* Thaw<ConfigT>::thaw_fast(stackChunkOop chunk) {
1914
assert(chunk == _cont.tail(), "");
1915
assert(!chunk->has_mixed_frames(), "");
1916
assert(!chunk->requires_barriers(), "");
1917
assert(!chunk->has_bitmap(), "");
1918
assert(!_thread->is_interp_only_mode(), "");
1920
LogTarget(Trace, continuations) lt;
1921
if (lt.develop_is_enabled()) {
1923
ls.print_cr("thaw_fast");
1924
chunk->print_on(true, &ls);
1927
// Below this heuristic, we thaw the whole chunk, above it we thaw just one frame.
1928
static const int threshold = 500; // words
1930
const int full_chunk_size = chunk->stack_size() - chunk->sp(); // this initial size could be reduced if it's a partial thaw
1931
int argsize, thaw_size;
1933
intptr_t* const chunk_sp = chunk->start_address() + chunk->sp();
1935
bool partial, empty;
1936
if (LIKELY(!TEST_THAW_ONE_CHUNK_FRAME && (full_chunk_size < threshold))) {
1937
prefetch_chunk_pd(chunk->start_address(), full_chunk_size); // prefetch anticipating memcpy starting at highest address
1940
argsize = chunk->argsize(); // must be called *before* clearing the chunk
1942
thaw_size = full_chunk_size;
1944
} else { // thaw a single frame
1946
thaw_size = remove_top_compiled_frame_from_chunk(chunk, argsize);
1947
empty = chunk->is_empty();
1950
// Are we thawing the last frame(s) in the continuation
1951
const bool is_last = empty && chunk->parent() == nullptr;
1952
assert(!is_last || argsize == 0, "");
1954
log_develop_trace(continuations)("thaw_fast partial: %d is_last: %d empty: %d size: %d argsize: %d entrySP: " PTR_FORMAT,
1955
partial, is_last, empty, thaw_size, argsize, p2i(_cont.entrySP()));
1957
ReconstructedStack rs(_cont.entrySP(), thaw_size, argsize);
1959
// also copy metadata words at frame bottom
1960
copy_from_chunk(chunk_sp - frame::metadata_words_at_bottom, rs.top(), rs.total_size());
1962
// update the ContinuationEntry
1963
_cont.set_argsize(argsize);
1964
log_develop_trace(continuations)("setting entry argsize: %d", _cont.argsize());
1965
assert(rs.bottom_sp() == _cont.entry()->bottom_sender_sp(), "");
1967
// install the return barrier if not last frame, or the entry's pc if last
1968
patch_return(rs.bottom_sp(), is_last);
1970
// insert the back links from callee to caller frames
1971
patch_caller_links(rs.top(), rs.top() + rs.total_size());
1973
assert(is_last == _cont.is_empty(), "");
1974
assert(_cont.chunk_invariant(), "");
1977
EventContinuationThawFast e;
1978
if (e.should_commit()) {
1979
e.set_id(cast_from_oop<u8>(chunk));
1980
e.set_size(thaw_size << LogBytesPerWord);
1981
e.set_full(!partial);
1987
set_anchor(_thread, rs.sp());
1988
log_frames(_thread);
1989
if (LoomDeoptAfterThaw) {
1990
do_deopt_after_thaw(_thread);
1992
clear_anchor(_thread);
1998
inline bool ThawBase::seen_by_gc() {
1999
return _barriers || _cont.tail()->is_gc_mode();
2002
NOINLINE intptr_t* ThawBase::thaw_slow(stackChunkOop chunk, bool return_barrier) {
2003
LogTarget(Trace, continuations) lt;
2004
if (lt.develop_is_enabled()) {
2006
ls.print_cr("thaw slow return_barrier: %d " INTPTR_FORMAT, return_barrier, p2i(chunk));
2007
chunk->print_on(true, &ls);
2011
EventContinuationThawSlow e;
2012
if (e.should_commit()) {
2013
e.set_id(cast_from_oop<u8>(_cont.continuation()));
2018
DEBUG_ONLY(_frames = 0;)
2020
int num_frames = (return_barrier ? 1 : 2);
2022
_stream = StackChunkFrameStream<ChunkFrames::Mixed>(chunk);
2023
_top_unextended_sp_before_thaw = _stream.unextended_sp();
2025
frame heap_frame = _stream.to_frame();
2026
if (lt.develop_is_enabled()) {
2028
ls.print_cr("top hframe before (thaw):");
2029
assert(heap_frame.is_heap_frame(), "should have created a relative frame");
2030
heap_frame.print_value_on(&ls, nullptr);
2033
#if INCLUDE_ZGC || INCLUDE_SHENANDOAHGC
2034
if (UseZGC || UseShenandoahGC) {
2035
_cont.tail()->relativize_derived_pointers_concurrently();
2039
frame caller; // the thawed caller on the stack
2040
recurse_thaw(heap_frame, caller, num_frames, true);
2041
finish_thaw(caller); // caller is now the topmost thawed frame
2044
assert(_cont.chunk_invariant(), "");
2046
JVMTI_ONLY(if (!return_barrier) invalidate_jvmti_stack(_thread));
2048
_thread->set_cont_fastpath(_fastpath);
2050
intptr_t* sp = caller.sp();
2054
void ThawBase::recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top) {
2055
log_develop_debug(continuations)("thaw num_frames: %d", num_frames);
2056
assert(!_cont.is_empty(), "no more frames");
2057
assert(num_frames > 0, "");
2058
assert(!heap_frame.is_empty(), "");
2060
if (top && heap_frame.is_safepoint_blob_frame()) {
2061
assert(ContinuationHelper::Frame::is_stub(heap_frame.cb()), "cb: %s", heap_frame.cb()->name());
2062
recurse_thaw_stub_frame(heap_frame, caller, num_frames);
2063
} else if (!heap_frame.is_interpreted_frame()) {
2064
recurse_thaw_compiled_frame(heap_frame, caller, num_frames, false);
2066
recurse_thaw_interpreted_frame(heap_frame, caller, num_frames);
2070
template<typename FKind>
2071
bool ThawBase::recurse_thaw_java_frame(frame& caller, int num_frames) {
2072
assert(num_frames > 0, "");
2074
DEBUG_ONLY(_frames++;)
2076
int argsize = _stream.stack_argsize();
2078
_stream.next(SmallRegisterMap::instance);
2079
assert(_stream.to_frame().is_empty() == _stream.is_done(), "");
2081
// we never leave a compiled caller of an interpreted frame as the top frame in the chunk
2082
// as it makes detecting that situation and adjusting unextended_sp tricky
2083
if (num_frames == 1 && !_stream.is_done() && FKind::interpreted && _stream.is_compiled()) {
2084
log_develop_trace(continuations)("thawing extra compiled frame to not leave a compiled interpreted-caller at top");
2088
if (num_frames == 1 || _stream.is_done()) { // end recursion
2089
finalize_thaw(caller, FKind::interpreted ? 0 : argsize);
2090
return true; // bottom
2092
recurse_thaw(_stream.to_frame(), caller, num_frames - 1, false);
2097
void ThawBase::finalize_thaw(frame& entry, int argsize) {
2098
stackChunkOop chunk = _cont.tail();
2100
if (!_stream.is_done()) {
2101
assert(_stream.sp() >= chunk->sp_address(), "");
2102
chunk->set_sp(chunk->to_offset(_stream.sp()));
2103
chunk->set_pc(_stream.pc());
2105
chunk->set_sp(chunk->bottom());
2106
chunk->set_pc(nullptr);
2108
assert(_stream.is_done() == chunk->is_empty(), "");
2110
int total_thawed = pointer_delta_as_int(_stream.unextended_sp(), _top_unextended_sp_before_thaw);
2111
chunk->set_max_thawing_size(chunk->max_thawing_size() - total_thawed);
2113
_cont.set_argsize(argsize);
2114
entry = new_entry_frame();
2116
assert(entry.sp() == _cont.entrySP(), "");
2117
assert(Continuation::is_continuation_enterSpecial(entry), "");
2118
assert(_cont.is_entry_frame(entry), "");
2121
inline void ThawBase::before_thaw_java_frame(const frame& hf, const frame& caller, bool bottom, int num_frame) {
2122
LogTarget(Trace, continuations) lt;
2123
if (lt.develop_is_enabled()) {
2125
ls.print_cr("======== THAWING FRAME: %d", num_frame);
2126
assert(hf.is_heap_frame(), "should be");
2127
hf.print_value_on(&ls, nullptr);
2129
assert(bottom == _cont.is_entry_frame(caller), "bottom: %d is_entry_frame: %d", bottom, _cont.is_entry_frame(hf));
2132
inline void ThawBase::after_thaw_java_frame(const frame& f, bool bottom) {
2134
LogTarget(Trace, continuations) lt;
2135
if (lt.develop_is_enabled()) {
2137
ls.print_cr("thawed frame:");
2138
print_frame_layout(f, false, &ls); // f.print_on(&ls);
2143
inline void ThawBase::patch(frame& f, const frame& caller, bool bottom) {
2144
assert(!bottom || caller.fp() == _cont.entryFP(), "");
2146
ContinuationHelper::Frame::patch_pc(caller, _cont.is_empty() ? caller.pc()
2147
: StubRoutines::cont_returnBarrier());
2149
// caller might have been deoptimized during thaw but we've overwritten the return address when copying f from the heap.
2150
// If the caller is not deoptimized, pc is unchanged.
2151
ContinuationHelper::Frame::patch_pc(caller, caller.raw_pc());
2154
patch_pd(f, caller);
2156
if (f.is_interpreted_frame()) {
2157
ContinuationHelper::InterpretedFrame::patch_sender_sp(f, caller);
2160
assert(!bottom || !_cont.is_empty() || Continuation::is_continuation_entry_frame(f, nullptr), "");
2161
assert(!bottom || (_cont.is_empty() != Continuation::is_cont_barrier_frame(f)), "");
2164
void ThawBase::clear_bitmap_bits(address start, address end) {
2165
assert(is_aligned(start, wordSize), "should be aligned: " PTR_FORMAT, p2i(start));
2166
assert(is_aligned(end, VMRegImpl::stack_slot_size), "should be aligned: " PTR_FORMAT, p2i(end));
2168
// we need to clear the bits that correspond to arguments as they reside in the caller frame
2169
// or they will keep objects that are otherwise unreachable alive.
2171
// Align `end` if UseCompressedOops is not set to avoid UB when calculating the bit index, since
2172
// `end` could be at an odd number of stack slots from `start`, i.e might not be oop aligned.
2173
// If that's the case the bit range corresponding to the last stack slot should not have bits set
2174
// anyways and we assert that before returning.
2175
address effective_end = UseCompressedOops ? end : align_down(end, wordSize);
2176
log_develop_trace(continuations)("clearing bitmap for " INTPTR_FORMAT " - " INTPTR_FORMAT, p2i(start), p2i(effective_end));
2177
stackChunkOop chunk = _cont.tail();
2178
chunk->bitmap().clear_range(chunk->bit_index_for(start), chunk->bit_index_for(effective_end));
2179
assert(effective_end == end || !chunk->bitmap().at(chunk->bit_index_for(effective_end)), "bit should not be set");
2182
NOINLINE void ThawBase::recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames) {
2183
assert(hf.is_interpreted_frame(), "");
2185
if (UNLIKELY(seen_by_gc())) {
2186
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance);
2189
const bool is_bottom_frame = recurse_thaw_java_frame<ContinuationHelper::InterpretedFrame>(caller, num_frames);
2191
DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);)
2193
_align_size += frame::align_wiggle; // possible added alignment for internal interpreted frame alignment om AArch64
2195
frame f = new_stack_frame<ContinuationHelper::InterpretedFrame>(hf, caller, is_bottom_frame);
2197
intptr_t* const stack_frame_top = f.sp() + frame::metadata_words_at_top;
2198
intptr_t* const stack_frame_bottom = ContinuationHelper::InterpretedFrame::frame_bottom(f);
2199
intptr_t* const heap_frame_top = hf.unextended_sp() + frame::metadata_words_at_top;
2200
intptr_t* const heap_frame_bottom = ContinuationHelper::InterpretedFrame::frame_bottom(hf);
2202
assert(hf.is_heap_frame(), "should be");
2203
assert(!f.is_heap_frame(), "should not be");
2205
const int fsize = pointer_delta_as_int(heap_frame_bottom, heap_frame_top);
2206
assert((stack_frame_bottom == stack_frame_top + fsize), "");
2208
// Some architectures (like AArch64/PPC64/RISC-V) add padding between the locals and the fixed_frame to keep the fp 16-byte-aligned.
2209
// On those architectures we freeze the padding in order to keep the same fp-relative offsets in the fixed_frame.
2210
copy_from_chunk(heap_frame_top, stack_frame_top, fsize);
2212
// Make sure the relativized locals is already set.
2213
assert(f.interpreter_frame_local_at(0) == stack_frame_bottom - 1, "invalid frame bottom");
2215
derelativize_interpreted_frame_metadata(hf, f);
2216
patch(f, caller, is_bottom_frame);
2218
assert(f.is_interpreted_frame_valid(_cont.thread()), "invalid thawed frame");
2219
assert(stack_frame_bottom <= ContinuationHelper::Frame::frame_top(caller), "");
2221
CONT_JFR_ONLY(_jfr_info.record_interpreted_frame();)
2223
maybe_set_fastpath(f.sp());
2225
const int locals = hf.interpreter_frame_method()->max_locals();
2227
if (!is_bottom_frame) {
2228
// can only fix caller once this frame is thawed (due to callee saved regs)
2229
_cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance);
2230
} else if (_cont.tail()->has_bitmap() && locals > 0) {
2231
assert(hf.is_heap_frame(), "should be");
2232
address start = (address)(heap_frame_bottom - locals);
2233
address end = (address)heap_frame_bottom;
2234
clear_bitmap_bits(start, end);
2237
DEBUG_ONLY(after_thaw_java_frame(f, is_bottom_frame);)
2241
void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller) {
2242
assert(!hf.is_interpreted_frame(), "");
2243
assert(_cont.is_preempted() || !stub_caller, "stub caller not at preemption");
2245
if (!stub_caller && UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap
2246
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, SmallRegisterMap::instance);
2249
const bool is_bottom_frame = recurse_thaw_java_frame<ContinuationHelper::CompiledFrame>(caller, num_frames);
2251
DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);)
2253
assert(caller.sp() == caller.unextended_sp(), "");
2255
if ((!is_bottom_frame && caller.is_interpreted_frame()) || (is_bottom_frame && Interpreter::contains(_cont.tail()->pc()))) {
2256
_align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in freeze_interpreted_frame
2259
// new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not
2260
// yet laid out in the stack, and so the original_pc is not stored in it.
2261
// As a result, f.is_deoptimized_frame() is always false and we must test hf to know if the frame is deoptimized.
2262
frame f = new_stack_frame<ContinuationHelper::CompiledFrame>(hf, caller, is_bottom_frame);
2263
intptr_t* const stack_frame_top = f.sp();
2264
intptr_t* const heap_frame_top = hf.unextended_sp();
2266
const int added_argsize = (is_bottom_frame || caller.is_interpreted_frame()) ? hf.compiled_frame_stack_argsize() : 0;
2267
int fsize = ContinuationHelper::CompiledFrame::size(hf) + added_argsize;
2268
assert(fsize <= (int)(caller.unextended_sp() - f.unextended_sp()), "");
2270
intptr_t* from = heap_frame_top - frame::metadata_words_at_bottom;
2271
intptr_t* to = stack_frame_top - frame::metadata_words_at_bottom;
2272
// copy metadata, except the metadata at the top of the (unextended) entry frame
2273
int sz = fsize + frame::metadata_words_at_bottom + (is_bottom_frame && added_argsize == 0 ? 0 : frame::metadata_words_at_top);
2275
// If we're the bottom-most thawed frame, we're writing to within one word from entrySP
2276
// (we might have one padding word for alignment)
2277
assert(!is_bottom_frame || (_cont.entrySP() - 1 <= to + sz && to + sz <= _cont.entrySP()), "");
2278
assert(!is_bottom_frame || hf.compiled_frame_stack_argsize() != 0 || (to + sz && to + sz == _cont.entrySP()), "");
2280
copy_from_chunk(from, to, sz); // copying good oops because we invoked barriers above
2282
patch(f, caller, is_bottom_frame);
2284
// f.is_deoptimized_frame() is always false and we must test hf.is_deoptimized_frame() (see comment above)
2285
assert(!f.is_deoptimized_frame(), "");
2286
if (hf.is_deoptimized_frame()) {
2287
maybe_set_fastpath(f.sp());
2288
} else if (_thread->is_interp_only_mode()
2289
|| (_cont.is_preempted() && f.cb()->as_nmethod()->is_marked_for_deoptimization())) {
2290
// The caller of the safepoint stub when the continuation is preempted is not at a call instruction, and so
2291
// cannot rely on nmethod patching for deopt.
2292
assert(_thread->is_interp_only_mode() || stub_caller, "expected a stub-caller");
2294
log_develop_trace(continuations)("Deoptimizing thawed frame");
2295
DEBUG_ONLY(ContinuationHelper::Frame::patch_pc(f, nullptr));
2297
f.deoptimize(nullptr); // the null thread simply avoids the assertion in deoptimize which we're not set up for
2298
assert(f.is_deoptimized_frame(), "");
2299
assert(ContinuationHelper::Frame::is_deopt_return(f.raw_pc(), f), "");
2300
maybe_set_fastpath(f.sp());
2303
if (!is_bottom_frame) {
2304
// can only fix caller once this frame is thawed (due to callee saved regs); this happens on the stack
2305
_cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance);
2306
} else if (_cont.tail()->has_bitmap() && added_argsize > 0) {
2307
address start = (address)(heap_frame_top + ContinuationHelper::CompiledFrame::size(hf) + frame::metadata_words_at_top);
2308
int stack_args_slots = f.cb()->as_nmethod()->num_stack_arg_slots(false /* rounded */);
2309
int argsize_in_bytes = stack_args_slots * VMRegImpl::stack_slot_size;
2310
clear_bitmap_bits(start, start + argsize_in_bytes);
2313
DEBUG_ONLY(after_thaw_java_frame(f, is_bottom_frame);)
2317
void ThawBase::recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames) {
2318
DEBUG_ONLY(_frames++;)
2321
RegisterMap map(nullptr,
2322
RegisterMap::UpdateMap::include,
2323
RegisterMap::ProcessFrames::skip,
2324
RegisterMap::WalkContinuation::skip);
2325
map.set_include_argument_oops(false);
2327
assert(!_stream.is_done(), "");
2328
if (UNLIKELY(seen_by_gc())) { // we're now doing this on the stub's caller
2329
_cont.tail()->do_barriers<stackChunkOopDesc::BarrierType::Store>(_stream, &map);
2331
assert(!_stream.is_done(), "");
2334
recurse_thaw_compiled_frame(_stream.to_frame(), caller, num_frames, true); // this could be deoptimized
2336
DEBUG_ONLY(before_thaw_java_frame(hf, caller, false, num_frames);)
2338
assert(ContinuationHelper::Frame::is_stub(hf.cb()), "");
2339
assert(caller.sp() == caller.unextended_sp(), "");
2340
assert(!caller.is_interpreted_frame(), "");
2342
int fsize = ContinuationHelper::StubFrame::size(hf);
2344
frame f = new_stack_frame<ContinuationHelper::StubFrame>(hf, caller, false);
2345
intptr_t* stack_frame_top = f.sp();
2346
intptr_t* heap_frame_top = hf.sp();
2348
copy_from_chunk(heap_frame_top - frame::metadata_words, stack_frame_top - frame::metadata_words,
2349
fsize + frame::metadata_words);
2351
{ // can only fix caller once this frame is thawed (due to callee saved regs)
2352
RegisterMap map(nullptr,
2353
RegisterMap::UpdateMap::include,
2354
RegisterMap::ProcessFrames::skip,
2355
RegisterMap::WalkContinuation::skip); // map.clear();
2356
map.set_include_argument_oops(false);
2357
f.oop_map()->update_register_map(&f, &map);
2358
ContinuationHelper::update_register_map_with_callee(caller, &map);
2359
_cont.tail()->fix_thawed_frame(caller, &map);
2362
DEBUG_ONLY(after_thaw_java_frame(f, false);)
2366
void ThawBase::finish_thaw(frame& f) {
2367
stackChunkOop chunk = _cont.tail();
2369
if (chunk->is_empty()) {
2370
// Only remove chunk from list if it can't be reused for another freeze
2372
_cont.set_tail(chunk->parent());
2374
chunk->set_has_mixed_frames(false);
2376
chunk->set_max_thawing_size(0);
2378
chunk->set_max_thawing_size(chunk->max_thawing_size() - _align_size);
2380
assert(chunk->is_empty() == (chunk->max_thawing_size() == 0), "");
2382
if (!is_aligned(f.sp(), frame::frame_alignment)) {
2383
assert(f.is_interpreted_frame(), "");
2384
f.set_sp(align_down(f.sp(), frame::frame_alignment));
2386
push_return_frame(f);
2387
chunk->fix_thawed_frame(f, SmallRegisterMap::instance); // can only fix caller after push_return_frame (due to callee saved regs)
2389
assert(_cont.is_empty() == _cont.last_frame().is_empty(), "");
2391
log_develop_trace(continuations)("thawed %d frames", _frames);
2393
LogTarget(Trace, continuations) lt;
2394
if (lt.develop_is_enabled()) {
2396
ls.print_cr("top hframe after (thaw):");
2397
_cont.last_frame().print_value_on(&ls, nullptr);
2401
void ThawBase::push_return_frame(frame& f) { // see generate_cont_thaw
2402
assert(!f.is_compiled_frame() || f.is_deoptimized_frame() == f.cb()->as_nmethod()->is_deopt_pc(f.raw_pc()), "");
2403
assert(!f.is_compiled_frame() || f.is_deoptimized_frame() == (f.pc() != f.raw_pc()), "");
2405
LogTarget(Trace, continuations) lt;
2406
if (lt.develop_is_enabled()) {
2408
ls.print_cr("push_return_frame");
2409
f.print_value_on(&ls, nullptr);
2412
assert(f.sp() - frame::metadata_words_at_bottom >= _top_stack_address, "overwrote past thawing space"
2413
" to: " INTPTR_FORMAT " top_address: " INTPTR_FORMAT, p2i(f.sp() - frame::metadata_words), p2i(_top_stack_address));
2414
ContinuationHelper::Frame::patch_pc(f, f.raw_pc()); // in case we want to deopt the frame in a full transition, this is checked.
2415
ContinuationHelper::push_pd(f);
2417
assert(ContinuationHelper::Frame::assert_frame_laid_out(f), "");
2420
// returns new top sp
2421
// called after preparations (stack overflow check and making room)
2422
template<typename ConfigT>
2423
static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::thaw_kind kind) {
2424
assert(thread == JavaThread::current(), "Must be current thread");
2426
CONT_JFR_ONLY(EventContinuationThaw event;)
2428
log_develop_trace(continuations)("~~~~ thaw kind: %d sp: " INTPTR_FORMAT, kind, p2i(thread->last_continuation()->entry_sp()));
2430
ContinuationEntry* entry = thread->last_continuation();
2431
assert(entry != nullptr, "");
2432
oop oopCont = entry->cont_oop(thread);
2434
assert(!jdk_internal_vm_Continuation::done(oopCont), "");
2435
assert(oopCont == get_continuation(thread), "");
2436
verify_continuation(oopCont);
2438
assert(entry->is_virtual_thread() == (entry->scope(thread) == java_lang_VirtualThread::vthread_scope()), "");
2440
ContinuationWrapper cont(thread, oopCont);
2441
log_develop_debug(continuations)("THAW #" INTPTR_FORMAT " " INTPTR_FORMAT, cont.hash(), p2i((oopDesc*)oopCont));
2444
set_anchor_to_entry(thread, cont.entry());
2446
clear_anchor(thread);
2449
Thaw<ConfigT> thw(thread, cont);
2450
intptr_t* const sp = thw.thaw(kind);
2451
assert(is_aligned(sp, frame::frame_alignment), "");
2453
// All or part of the frames have been thawed so we know they don't hold any monitors except JNI monitors.
2454
assert(thread->held_monitor_count() == thread->jni_monitor_count(), "Must be");
2458
set_anchor(thread, sp0);
2460
if (LoomVerifyAfterThaw) {
2461
assert(do_verify_after_thaw(thread, cont.tail(), tty), "");
2463
assert(ContinuationEntry::assert_entry_frame_laid_out(thread), "");
2464
clear_anchor(thread);
2466
LogTarget(Trace, continuations) lt;
2467
if (lt.develop_is_enabled()) {
2469
ls.print_cr("Jumping to frame (thaw):");
2470
frame(sp).print_value_on(&ls, nullptr);
2474
CONT_JFR_ONLY(thw.jfr_info().post_jfr_event(&event, cont.continuation(), thread);)
2476
verify_continuation(cont.continuation());
2477
log_develop_debug(continuations)("=== End of thaw #" INTPTR_FORMAT, cont.hash());
2483
static void do_deopt_after_thaw(JavaThread* thread) {
2485
StackFrameStream fst(thread, true, false);
2486
fst.register_map()->set_include_argument_oops(false);
2487
ContinuationHelper::update_register_map_with_callee(*fst.current(), fst.register_map());
2488
for (; !fst.is_done(); fst.next()) {
2489
if (fst.current()->cb()->is_nmethod()) {
2490
nmethod* nm = fst.current()->cb()->as_nmethod();
2491
if (!nm->method()->is_continuation_native_intrinsic()) {
2492
nm->make_deoptimized();
2498
class ThawVerifyOopsClosure: public OopClosure {
2501
bool is_good_oop(oop o) {
2502
return dbg_is_safe(o, -1) && dbg_is_safe(o->klass(), -1) && oopDesc::is_oop(o) && o->klass()->is_klass();
2505
ThawVerifyOopsClosure(outputStream* st) : _p(nullptr), _st(st) {}
2506
intptr_t* p() { return _p; }
2507
void reset() { _p = nullptr; }
2509
virtual void do_oop(oop* p) {
2511
if (o == nullptr || is_good_oop(o)) {
2515
_st->print_cr("*** non-oop " PTR_FORMAT " found at " PTR_FORMAT, p2i(*p), p2i(p));
2517
virtual void do_oop(narrowOop* p) {
2518
oop o = RawAccess<>::oop_load(p);
2519
if (o == nullptr || is_good_oop(o)) {
2523
_st->print_cr("*** (narrow) non-oop %x found at " PTR_FORMAT, (int)(*p), p2i(p));
2527
static bool do_verify_after_thaw(JavaThread* thread, stackChunkOop chunk, outputStream* st) {
2528
assert(thread->has_last_Java_frame(), "");
2531
ThawVerifyOopsClosure cl(st);
2532
NMethodToOopClosure cf(&cl, false);
2534
StackFrameStream fst(thread, true, false);
2535
fst.register_map()->set_include_argument_oops(false);
2536
ContinuationHelper::update_register_map_with_callee(*fst.current(), fst.register_map());
2537
for (; !fst.is_done() && !Continuation::is_continuation_enterSpecial(*fst.current()); fst.next()) {
2538
if (fst.current()->cb()->is_nmethod() && fst.current()->cb()->as_nmethod()->is_marked_for_deoptimization()) {
2539
st->print_cr(">>> do_verify_after_thaw deopt");
2540
fst.current()->deoptimize(nullptr);
2541
fst.current()->print_on(st);
2544
fst.current()->oops_do(&cl, &cf, fst.register_map());
2545
if (cl.p() != nullptr) {
2546
frame fr = *fst.current();
2547
st->print_cr("Failed for frame barriers: %d",chunk->requires_barriers());
2549
if (!fr.is_interpreted_frame()) {
2550
st->print_cr("size: %d argsize: %d",
2551
ContinuationHelper::NonInterpretedUnknownFrame::size(fr),
2552
ContinuationHelper::NonInterpretedUnknownFrame::stack_argsize(fr));
2554
VMReg reg = fst.register_map()->find_register_spilled_here(cl.p(), fst.current()->sp());
2555
if (reg != nullptr) {
2556
st->print_cr("Reg %s %d", reg->name(), reg->is_stack() ? (int)reg->reg2stack() : -99);
2559
DEBUG_ONLY(thread->print_frame_layout();)
2560
if (chunk != nullptr) {
2561
chunk->print_on(true, st);
2569
static void log_frames(JavaThread* thread) {
2570
const static int show_entry_callers = 3;
2571
LogTarget(Trace, continuations) lt;
2572
if (!lt.develop_is_enabled()) {
2577
ls.print_cr("------- frames ---------");
2578
if (!thread->has_last_Java_frame()) {
2579
ls.print_cr("NO ANCHOR!");
2582
RegisterMap map(thread,
2583
RegisterMap::UpdateMap::include,
2584
RegisterMap::ProcessFrames::include,
2585
RegisterMap::WalkContinuation::skip);
2586
map.set_include_argument_oops(false);
2589
for (frame f = thread->last_frame(); !f.is_entry_frame(); f = f.sender(&map)) {
2593
map.set_skip_missing(true);
2594
ResetNoHandleMark rnhm;
2596
HandleMark hm(Thread::current());
2600
int post_entry = -1;
2601
for (frame f = thread->last_frame(); !f.is_entry_frame(); f = f.sender(&map)) {
2602
f.describe(values, i++, &map);
2603
if (post_entry >= 0 || Continuation::is_continuation_enterSpecial(f))
2605
if (post_entry >= show_entry_callers)
2608
values.print_on(thread, &ls);
2611
ls.print_cr("======= end frames =========");
2615
#include CPU_HEADER_INLINE(continuationFreezeThaw)
2618
static void print_frame_layout(const frame& f, bool callee_complete, outputStream* st) {
2621
assert(f.get_cb() != nullptr, "");
2622
RegisterMap map(f.is_heap_frame() ?
2624
JavaThread::current(),
2625
RegisterMap::UpdateMap::include,
2626
RegisterMap::ProcessFrames::skip,
2627
RegisterMap::WalkContinuation::skip);
2628
map.set_include_argument_oops(false);
2629
map.set_skip_missing(true);
2630
if (callee_complete) {
2631
frame::update_map_with_saved_link(&map, ContinuationHelper::Frame::callee_link_address(f));
2633
const_cast<frame&>(f).describe(values, 0, &map);
2634
values.print_on(static_cast<JavaThread*>(nullptr), st);
2638
static address thaw_entry = nullptr;
2639
static address freeze_entry = nullptr;
2641
address Continuation::thaw_entry() {
2642
return ::thaw_entry;
2645
address Continuation::freeze_entry() {
2646
return ::freeze_entry;
2649
class ConfigResolve {
2651
static void resolve() { resolve_compressed(); }
2653
static void resolve_compressed() {
2654
UseCompressedOops ? resolve_gc<true>()
2655
: resolve_gc<false>();
2659
template <bool use_compressed>
2660
static void resolve_gc() {
2661
BarrierSet* bs = BarrierSet::barrier_set();
2662
assert(bs != nullptr, "freeze/thaw invoked before BarrierSet is set");
2663
switch (bs->kind()) {
2664
#define BARRIER_SET_RESOLVE_BARRIER_CLOSURE(bs_name) \
2665
case BarrierSet::bs_name: { \
2666
resolve<use_compressed, typename BarrierSet::GetType<BarrierSet::bs_name>::type>(); \
2669
FOR_EACH_CONCRETE_BARRIER_SET_DO(BARRIER_SET_RESOLVE_BARRIER_CLOSURE)
2670
#undef BARRIER_SET_RESOLVE_BARRIER_CLOSURE
2673
fatal("BarrierSet resolving not implemented");
2677
template <bool use_compressed, typename BarrierSetT>
2678
static void resolve() {
2679
typedef Config<use_compressed ? oop_kind::NARROW : oop_kind::WIDE, BarrierSetT> SelectedConfigT;
2681
freeze_entry = (address)freeze<SelectedConfigT>;
2683
// If we wanted, we could templatize by kind and have three different thaw entries
2684
thaw_entry = (address)thaw<SelectedConfigT>;
2688
void Continuation::init() {
2689
ConfigResolve::resolve();