jdk

1
/*
2
 * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
3
 * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
4
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5
 *
6
 * This code is free software; you can redistribute it and/or modify it
7
 * under the terms of the GNU General Public License version 2 only, as
8
 * published by the Free Software Foundation.
9
 *
10
 * This code is distributed in the hope that it will be useful, but WITHOUT
11
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
13
 * version 2 for more details (a copy is included in the LICENSE file that
14
 * accompanied this code).
15
 *
16
 * You should have received a copy of the GNU General Public License version
17
 * 2 along with this work; if not, write to the Free Software Foundation,
18
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19
 *
20
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21
 * or visit www.oracle.com if you need additional information or have any
22
 * questions.
23
 *
24
 */
25

26
#include "precompiled.hpp"
27
#include "cds/cdsConfig.hpp"
28
#include "classfile/javaClasses.hpp"
29
#include "classfile/javaThreadStatus.hpp"
30
#include "gc/shared/barrierSet.hpp"
31
#include "jfr/jfrEvents.hpp"
32
#include "jvm.h"
33
#include "jvmtifiles/jvmtiEnv.hpp"
34
#include "logging/log.hpp"
35
#include "memory/allocation.inline.hpp"
36
#include "memory/iterator.hpp"
37
#include "memory/resourceArea.hpp"
38
#include "nmt/memTracker.hpp"
39
#include "oops/oop.inline.hpp"
40
#include "runtime/atomic.hpp"
41
#include "runtime/handles.inline.hpp"
42
#include "runtime/javaThread.inline.hpp"
43
#include "runtime/nonJavaThread.hpp"
44
#include "runtime/orderAccess.hpp"
45
#include "runtime/osThread.hpp"
46
#include "runtime/safepoint.hpp"
47
#include "runtime/safepointMechanism.inline.hpp"
48
#include "runtime/thread.inline.hpp"
49
#include "runtime/threadSMR.inline.hpp"
50
#include "utilities/macros.hpp"
51
#include "utilities/spinYield.hpp"
52
#if INCLUDE_JFR
53
#include "jfr/jfr.hpp"
54
#endif
55

56
#ifndef USE_LIBRARY_BASED_TLS_ONLY
57
// Current thread is maintained as a thread-local variable
58
THREAD_LOCAL Thread* Thread::_thr_current = nullptr;
59
#endif
60

61
// ======= Thread ========
62
// Base class for all threads: VMThread, WatcherThread, ConcurrentMarkSweepThread,
63
// JavaThread
64

65
DEBUG_ONLY(Thread* Thread::_starting_thread = nullptr;)
66

67
Thread::Thread(MEMFLAGS flags) {
68

69
  DEBUG_ONLY(_run_state = PRE_CALL_RUN;)
70

71
  // stack and get_thread
72
  set_stack_base(nullptr);
73
  set_stack_size(0);
74
  set_lgrp_id(-1);
75
  DEBUG_ONLY(clear_suspendible_thread();)
76
  DEBUG_ONLY(clear_indirectly_suspendible_thread();)
77
  DEBUG_ONLY(clear_indirectly_safepoint_thread();)
78

79
  // allocated data structures
80
  set_osthread(nullptr);
81
  set_resource_area(new (flags) ResourceArea(flags));
82
  DEBUG_ONLY(_current_resource_mark = nullptr;)
83
  set_handle_area(new (flags) HandleArea(flags, nullptr));
84
  set_metadata_handles(new (mtClass) GrowableArray<Metadata*>(30, mtClass));
85
  set_last_handle_mark(nullptr);
86
  DEBUG_ONLY(_missed_ic_stub_refill_verifier = nullptr);
87

88
  // Initial value of zero ==> never claimed.
89
  _threads_do_token = 0;
90
  _threads_hazard_ptr = nullptr;
91
  _threads_list_ptr = nullptr;
92
  _nested_threads_hazard_ptr_cnt = 0;
93
  _rcu_counter = 0;
94

95
  // the handle mark links itself to last_handle_mark
96
  new HandleMark(this);
97

98
  // plain initialization
99
  debug_only(_owned_locks = nullptr;)
100
  NOT_PRODUCT(_skip_gcalot = false;)
101
  _jvmti_env_iteration_count = 0;
102
  set_allocated_bytes(0);
103
  _current_pending_raw_monitor = nullptr;
104
  _vm_error_callbacks = nullptr;
105

106
  // thread-specific hashCode stream generator state - Marsaglia shift-xor form
107
  // If we are dumping, keep ihashes constant. Note that during dumping we only
108
  // ever run one java thread, and no other thread should generate ihashes either,
109
  // so using a constant seed should work fine.
110
  _hashStateX = CDSConfig::is_dumping_static_archive() ? 0x12345678 : os::random();
111
  _hashStateY = 842502087;
112
  _hashStateZ = 0x8767;    // (int)(3579807591LL & 0xffff) ;
113
  _hashStateW = 273326509;
114

115
  // Many of the following fields are effectively final - immutable
116
  // Note that nascent threads can't use the Native Monitor-Mutex
117
  // construct until the _MutexEvent is initialized ...
118
  // CONSIDER: instead of using a fixed set of purpose-dedicated ParkEvents
119
  // we might instead use a stack of ParkEvents that we could provision on-demand.
120
  // The stack would act as a cache to avoid calls to ParkEvent::Allocate()
121
  // and ::Release()
122
  _ParkEvent   = ParkEvent::Allocate(this);
123

124
#ifdef CHECK_UNHANDLED_OOPS
125
  if (CheckUnhandledOops) {
126
    _unhandled_oops = new UnhandledOops(this);
127
  }
128
#endif // CHECK_UNHANDLED_OOPS
129

130
  // Notify the barrier set that a thread is being created. The initial
131
  // thread is created before the barrier set is available.  The call to
132
  // BarrierSet::on_thread_create() for this thread is therefore deferred
133
  // to BarrierSet::set_barrier_set().
134
  BarrierSet* const barrier_set = BarrierSet::barrier_set();
135
  if (barrier_set != nullptr) {
136
    barrier_set->on_thread_create(this);
137
  } else {
138
    // Only the main thread should be created before the barrier set
139
    // and that happens just before Thread::current is set. No other thread
140
    // can attach as the VM is not created yet, so they can't execute this code.
141
    // If the main thread creates other threads before the barrier set that is an error.
142
    assert(Thread::current_or_null() == nullptr, "creating thread before barrier set");
143
  }
144

145
  MACOS_AARCH64_ONLY(DEBUG_ONLY(_wx_init = false));
146
}
147

148
void Thread::initialize_tlab() {
149
  if (UseTLAB) {
150
    tlab().initialize();
151
  }
152
}
153

154
void Thread::initialize_thread_current() {
155
#ifndef USE_LIBRARY_BASED_TLS_ONLY
156
  assert(_thr_current == nullptr, "Thread::current already initialized");
157
  _thr_current = this;
158
#endif
159
  assert(ThreadLocalStorage::thread() == nullptr, "ThreadLocalStorage::thread already initialized");
160
  ThreadLocalStorage::set_thread(this);
161
  assert(Thread::current() == ThreadLocalStorage::thread(), "TLS mismatch!");
162
}
163

164
void Thread::clear_thread_current() {
165
  assert(Thread::current() == ThreadLocalStorage::thread(), "TLS mismatch!");
166
#ifndef USE_LIBRARY_BASED_TLS_ONLY
167
  _thr_current = nullptr;
168
#endif
169
  ThreadLocalStorage::set_thread(nullptr);
170
}
171

172
void Thread::record_stack_base_and_size() {
173
  // Note: at this point, Thread object is not yet initialized. Do not rely on
174
  // any members being initialized. Do not rely on Thread::current() being set.
175
  // If possible, refrain from doing anything which may crash or assert since
176
  // quite probably those crash dumps will be useless.
177
  address base;
178
  size_t size;
179
  os::current_stack_base_and_size(&base, &size);
180
  set_stack_base(base);
181
  set_stack_size(size);
182

183
  // Set stack limits after thread is initialized.
184
  if (is_Java_thread()) {
185
    JavaThread::cast(this)->stack_overflow_state()->initialize(stack_base(), stack_end());
186
  }
187
}
188

189
void Thread::register_thread_stack_with_NMT() {
190
  MemTracker::record_thread_stack(stack_end(), stack_size());
191
}
192

193
void Thread::unregister_thread_stack_with_NMT() {
194
  MemTracker::release_thread_stack(stack_end(), stack_size());
195
}
196

197
void Thread::call_run() {
198
  DEBUG_ONLY(_run_state = CALL_RUN;)
199

200
  // At this point, Thread object should be fully initialized and
201
  // Thread::current() should be set.
202

203
  assert(Thread::current_or_null() != nullptr, "current thread is unset");
204
  assert(Thread::current_or_null() == this, "current thread is wrong");
205

206
  // Perform common initialization actions
207

208
  MACOS_AARCH64_ONLY(this->init_wx());
209

210
  register_thread_stack_with_NMT();
211

212
  JFR_ONLY(Jfr::on_thread_start(this);)
213

214
  log_debug(os, thread)("Thread " UINTX_FORMAT " stack dimensions: "
215
    PTR_FORMAT "-" PTR_FORMAT " (" SIZE_FORMAT "k).",
216
    os::current_thread_id(), p2i(stack_end()),
217
    p2i(stack_base()), stack_size()/1024);
218

219
  // Perform <ChildClass> initialization actions
220
  DEBUG_ONLY(_run_state = PRE_RUN;)
221
  this->pre_run();
222

223
  // Invoke <ChildClass>::run()
224
  DEBUG_ONLY(_run_state = RUN;)
225
  this->run();
226
  // Returned from <ChildClass>::run(). Thread finished.
227

228
  // Perform common tear-down actions
229

230
  assert(Thread::current_or_null() != nullptr, "current thread is unset");
231
  assert(Thread::current_or_null() == this, "current thread is wrong");
232

233
  // Perform <ChildClass> tear-down actions
234
  DEBUG_ONLY(_run_state = POST_RUN;)
235
  this->post_run();
236

237
  // Note: at this point the thread object may already have deleted itself,
238
  // so from here on do not dereference *this*. Not all thread types currently
239
  // delete themselves when they terminate. But no thread should ever be deleted
240
  // asynchronously with respect to its termination - that is what _run_state can
241
  // be used to check.
242

243
  assert(Thread::current_or_null() == nullptr, "current thread still present");
244
}
245

246
Thread::~Thread() {
247

248
  // Attached threads will remain in PRE_CALL_RUN, as will threads that don't actually
249
  // get started due to errors etc. Any active thread should at least reach post_run
250
  // before it is deleted (usually in post_run()).
251
  assert(_run_state == PRE_CALL_RUN ||
252
         _run_state == POST_RUN, "Active Thread deleted before post_run(): "
253
         "_run_state=%d", (int)_run_state);
254

255
  // Notify the barrier set that a thread is being destroyed. Note that a barrier
256
  // set might not be available if we encountered errors during bootstrapping.
257
  BarrierSet* const barrier_set = BarrierSet::barrier_set();
258
  if (barrier_set != nullptr) {
259
    barrier_set->on_thread_destroy(this);
260
  }
261

262
  // deallocate data structures
263
  delete resource_area();
264
  // since the handle marks are using the handle area, we have to deallocated the root
265
  // handle mark before deallocating the thread's handle area,
266
  assert(last_handle_mark() != nullptr, "check we have an element");
267
  delete last_handle_mark();
268
  assert(last_handle_mark() == nullptr, "check we have reached the end");
269

270
  ParkEvent::Release(_ParkEvent);
271
  // Set to null as a termination indicator for has_terminated().
272
  Atomic::store(&_ParkEvent, (ParkEvent*)nullptr);
273

274
  delete handle_area();
275
  delete metadata_handles();
276

277
  // osthread() can be null, if creation of thread failed.
278
  if (osthread() != nullptr) os::free_thread(osthread());
279

280
  // Clear Thread::current if thread is deleting itself and it has not
281
  // already been done. This must be done before the memory is deallocated.
282
  // Needed to ensure JNI correctly detects non-attached threads.
283
  if (this == Thread::current_or_null()) {
284
    Thread::clear_thread_current();
285
  }
286

287
  CHECK_UNHANDLED_OOPS_ONLY(if (CheckUnhandledOops) delete unhandled_oops();)
288
}
289

290
#ifdef ASSERT
291
// A JavaThread is considered dangling if it not handshake-safe with respect to
292
// the current thread, it is not on a ThreadsList, or not at safepoint.
293
void Thread::check_for_dangling_thread_pointer(Thread *thread) {
294
  assert(!thread->is_Java_thread() ||
295
         JavaThread::cast(thread)->is_handshake_safe_for(Thread::current()) ||
296
         !JavaThread::cast(thread)->on_thread_list() ||
297
         SafepointSynchronize::is_at_safepoint() ||
298
         ThreadsSMRSupport::is_a_protected_JavaThread_with_lock(JavaThread::cast(thread)),
299
         "possibility of dangling Thread pointer");
300
}
301
#endif
302

303
// Is the target JavaThread protected by the calling Thread or by some other
304
// mechanism?
305
//
306
bool Thread::is_JavaThread_protected(const JavaThread* target) {
307
  Thread* current_thread = Thread::current();
308

309
  // Do the simplest check first:
310
  if (SafepointSynchronize::is_at_safepoint()) {
311
    // The target is protected since JavaThreads cannot exit
312
    // while we're at a safepoint.
313
    return true;
314
  }
315

316
  // If the target hasn't been started yet then it is trivially
317
  // "protected". We assume the caller is the thread that will do
318
  // the starting.
319
  if (target->osthread() == nullptr || target->osthread()->get_state() <= INITIALIZED) {
320
    return true;
321
  }
322

323
  // Now make the simple checks based on who the caller is:
324
  if (current_thread == target || Threads_lock->owner() == current_thread) {
325
    // Target JavaThread is self or calling thread owns the Threads_lock.
326
    // Second check is the same as Threads_lock->owner_is_self(),
327
    // but we already have the current thread so check directly.
328
    return true;
329
  }
330

331
  // Check the ThreadsLists associated with the calling thread (if any)
332
  // to see if one of them protects the target JavaThread:
333
  if (is_JavaThread_protected_by_TLH(target)) {
334
    return true;
335
  }
336

337
  // Use this debug code with -XX:+UseNewCode to diagnose locations that
338
  // are missing a ThreadsListHandle or other protection mechanism:
339
  // guarantee(!UseNewCode, "current_thread=" INTPTR_FORMAT " is not protecting target="
340
  //           INTPTR_FORMAT, p2i(current_thread), p2i(target));
341

342
  // Note: Since 'target' isn't protected by a TLH, the call to
343
  // target->is_handshake_safe_for() may crash, but we have debug bits so
344
  // we'll be able to figure out what protection mechanism is missing.
345
  assert(target->is_handshake_safe_for(current_thread), "JavaThread=" INTPTR_FORMAT
346
         " is not protected and not handshake safe.", p2i(target));
347

348
  // The target JavaThread is not protected so it is not safe to query:
349
  return false;
350
}
351

352
// Is the target JavaThread protected by a ThreadsListHandle (TLH) associated
353
// with the calling Thread?
354
//
355
bool Thread::is_JavaThread_protected_by_TLH(const JavaThread* target) {
356
  Thread* current_thread = Thread::current();
357

358
  // Check the ThreadsLists associated with the calling thread (if any)
359
  // to see if one of them protects the target JavaThread:
360
  for (SafeThreadsListPtr* stlp = current_thread->_threads_list_ptr;
361
       stlp != nullptr; stlp = stlp->previous()) {
362
    if (stlp->list()->includes(target)) {
363
      // The target JavaThread is protected by this ThreadsList:
364
      return true;
365
    }
366
  }
367

368
  // The target JavaThread is not protected by a TLH so it is not safe to query:
369
  return false;
370
}
371

372
void Thread::set_priority(Thread* thread, ThreadPriority priority) {
373
  debug_only(check_for_dangling_thread_pointer(thread);)
374
  // Can return an error!
375
  (void)os::set_priority(thread, priority);
376
}
377

378

379
void Thread::start(Thread* thread) {
380
  // Start is different from resume in that its safety is guaranteed by context or
381
  // being called from a Java method synchronized on the Thread object.
382
  if (thread->is_Java_thread()) {
383
    // Initialize the thread state to RUNNABLE before starting this thread.
384
    // Can not set it after the thread started because we do not know the
385
    // exact thread state at that time. It could be in MONITOR_WAIT or
386
    // in SLEEPING or some other state.
387
    java_lang_Thread::set_thread_status(JavaThread::cast(thread)->threadObj(),
388
                                        JavaThreadStatus::RUNNABLE);
389
  }
390
  os::start_thread(thread);
391
}
392

393
// GC Support
394
bool Thread::claim_par_threads_do(uintx claim_token) {
395
  uintx token = _threads_do_token;
396
  if (token != claim_token) {
397
    uintx res = Atomic::cmpxchg(&_threads_do_token, token, claim_token);
398
    if (res == token) {
399
      return true;
400
    }
401
    guarantee(res == claim_token, "invariant");
402
  }
403
  return false;
404
}
405

406
void Thread::oops_do_no_frames(OopClosure* f, NMethodClosure* cf) {
407
  // Do oop for ThreadShadow
408
  f->do_oop((oop*)&_pending_exception);
409
  handle_area()->oops_do(f);
410
}
411

412
// If the caller is a NamedThread, then remember, in the current scope,
413
// the given JavaThread in its _processed_thread field.
414
class RememberProcessedThread: public StackObj {
415
  NamedThread* _cur_thr;
416
public:
417
  RememberProcessedThread(Thread* thread) {
418
    Thread* self = Thread::current();
419
    if (self->is_Named_thread()) {
420
      _cur_thr = (NamedThread *)self;
421
      assert(_cur_thr->processed_thread() == nullptr, "nesting not supported");
422
      _cur_thr->set_processed_thread(thread);
423
    } else {
424
      _cur_thr = nullptr;
425
    }
426
  }
427

428
  ~RememberProcessedThread() {
429
    if (_cur_thr) {
430
      assert(_cur_thr->processed_thread() != nullptr, "nesting not supported");
431
      _cur_thr->set_processed_thread(nullptr);
432
    }
433
  }
434
};
435

436
void Thread::oops_do(OopClosure* f, NMethodClosure* cf) {
437
  // Record JavaThread to GC thread
438
  RememberProcessedThread rpt(this);
439
  oops_do_no_frames(f, cf);
440
  oops_do_frames(f, cf);
441
}
442

443
void Thread::metadata_handles_do(void f(Metadata*)) {
444
  // Only walk the Handles in Thread.
445
  if (metadata_handles() != nullptr) {
446
    for (int i = 0; i< metadata_handles()->length(); i++) {
447
      f(metadata_handles()->at(i));
448
    }
449
  }
450
}
451

452
void Thread::print_on(outputStream* st, bool print_extended_info) const {
453
  // get_priority assumes osthread initialized
454
  if (osthread() != nullptr) {
455
    int os_prio;
456
    if (os::get_native_priority(this, &os_prio) == OS_OK) {
457
      st->print("os_prio=%d ", os_prio);
458
    }
459

460
    st->print("cpu=%.2fms ",
461
              (double)os::thread_cpu_time(const_cast<Thread*>(this), true) / 1000000.0
462
              );
463
    st->print("elapsed=%.2fs ",
464
              (double)_statistical_info.getElapsedTime() / 1000.0
465
              );
466
    if (is_Java_thread() && (PrintExtendedThreadInfo || print_extended_info)) {
467
      size_t allocated_bytes = (size_t) const_cast<Thread*>(this)->cooked_allocated_bytes();
468
      st->print("allocated=" SIZE_FORMAT "%s ",
469
                byte_size_in_proper_unit(allocated_bytes),
470
                proper_unit_for_byte_size(allocated_bytes)
471
                );
472
      st->print("defined_classes=" INT64_FORMAT " ", _statistical_info.getDefineClassCount());
473
    }
474

475
    st->print("tid=" INTPTR_FORMAT " ", p2i(this));
476
    if (!is_Java_thread() || !JavaThread::cast(this)->is_vthread_mounted()) {
477
      osthread()->print_on(st);
478
    }
479
  }
480
  ThreadsSMRSupport::print_info_on(this, st);
481
  st->print(" ");
482
  debug_only(if (WizardMode) print_owned_locks_on(st);)
483
}
484

485
void Thread::print() const { print_on(tty); }
486

487
// Thread::print_on_error() is called by fatal error handler. Don't use
488
// any lock or allocate memory.
489
void Thread::print_on_error(outputStream* st, char* buf, int buflen) const {
490
  assert(!(is_Compiler_thread() || is_Java_thread()), "Can't call name() here if it allocates");
491

492
  st->print("%s \"%s\"", type_name(), name());
493

494
  OSThread* os_thr = osthread();
495
  if (os_thr != nullptr) {
496
    st->fill_to(67);
497
    if (os_thr->get_state() != ZOMBIE) {
498
      // Use raw field members for stack base/size as this could be
499
      // called before a thread has run enough to initialize them.
500
      st->print(" [id=%d, stack(" PTR_FORMAT "," PTR_FORMAT ") (" PROPERFMT ")]",
501
                osthread()->thread_id(), p2i(_stack_base - _stack_size), p2i(_stack_base),
502
                PROPERFMTARGS(_stack_size));
503
    } else {
504
      st->print(" terminated");
505
    }
506
  } else {
507
    st->print(" unknown state (no osThread)");
508
  }
509
  ThreadsSMRSupport::print_info_on(this, st);
510
}
511

512
void Thread::print_value_on(outputStream* st) const {
513
  if (is_Named_thread()) {
514
    st->print(" \"%s\" ", name());
515
  }
516
  st->print(INTPTR_FORMAT, p2i(this));   // print address
517
}
518

519
#ifdef ASSERT
520
void Thread::print_owned_locks_on(outputStream* st) const {
521
  Mutex* cur = _owned_locks;
522
  if (cur == nullptr) {
523
    st->print(" (no locks) ");
524
  } else {
525
    st->print_cr(" Locks owned:");
526
    while (cur) {
527
      cur->print_on(st);
528
      cur = cur->next();
529
    }
530
  }
531
}
532
#endif // ASSERT
533

534
bool Thread::set_as_starting_thread() {
535
  assert(_starting_thread == nullptr, "already initialized: "
536
         "_starting_thread=" INTPTR_FORMAT, p2i(_starting_thread));
537
  // NOTE: this must be called inside the main thread.
538
  DEBUG_ONLY(_starting_thread = this;)
539
  return os::create_main_thread(JavaThread::cast(this));
540
}
541

542
// Ad-hoc mutual exclusion primitives: SpinLock
543
//
544
// We employ SpinLocks _only for low-contention, fixed-length
545
// short-duration critical sections where we're concerned
546
// about native mutex_t or HotSpot Mutex:: latency.
547
//
548
// TODO-FIXME: ListLock should be of type SpinLock.
549
// We should make this a 1st-class type, integrated into the lock
550
// hierarchy as leaf-locks.  Critically, the SpinLock structure
551
// should have sufficient padding to avoid false-sharing and excessive
552
// cache-coherency traffic.
553

554

555
typedef volatile int SpinLockT;
556

557
void Thread::SpinAcquire(volatile int * adr, const char * LockName) {
558
  if (Atomic::cmpxchg(adr, 0, 1) == 0) {
559
    return;   // normal fast-path return
560
  }
561

562
  // Slow-path : We've encountered contention -- Spin/Yield/Block strategy.
563
  int ctr = 0;
564
  int Yields = 0;
565
  for (;;) {
566
    while (*adr != 0) {
567
      ++ctr;
568
      if ((ctr & 0xFFF) == 0 || !os::is_MP()) {
569
        if (Yields > 5) {
570
          os::naked_short_sleep(1);
571
        } else {
572
          os::naked_yield();
573
          ++Yields;
574
        }
575
      } else {
576
        SpinPause();
577
      }
578
    }
579
    if (Atomic::cmpxchg(adr, 0, 1) == 0) return;
580
  }
581
}
582

583
void Thread::SpinRelease(volatile int * adr) {
584
  assert(*adr != 0, "invariant");
585
  OrderAccess::fence();      // guarantee at least release consistency.
586
  // Roach-motel semantics.
587
  // It's safe if subsequent LDs and STs float "up" into the critical section,
588
  // but prior LDs and STs within the critical section can't be allowed
589
  // to reorder or float past the ST that releases the lock.
590
  // Loads and stores in the critical section - which appear in program
591
  // order before the store that releases the lock - must also appear
592
  // before the store that releases the lock in memory visibility order.
593
  // Conceptually we need a #loadstore|#storestore "release" MEMBAR before
594
  // the ST of 0 into the lock-word which releases the lock, so fence
595
  // more than covers this on all platforms.
596
  *adr = 0;
597
}
598