deps/v8/src/wasm/module-compiler.cc - main - githubmirror/node

deps/v8/src/wasm/module-compiler.cc
4 268 строк173 KB
Michaël Zasso
deps: update V8 to 14.6.202.33
24 апр 2026, 19:01
24 апр 2026, 19:01f1e0b83
// Copyright 2017 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/wasm/module-compiler.h"

#include <algorithm>
#include <atomic>
#include <memory>
#include <queue>

#include "src/api/api-inl.h"
#include "src/base/enum-set.h"
#include "src/base/fpu.h"
#include "src/base/platform/mutex.h"
#include "src/base/platform/semaphore.h"
#include "src/base/platform/time.h"
#include "src/codegen/compiler.h"
#include "src/compiler/wasm-compiler.h"
#include "src/debug/debug.h"
#include "src/handles/global-handles-inl.h"
#include "src/logging/counters-scopes.h"
#include "src/logging/metrics.h"
#include "src/tracing/trace-event.h"
#include "src/wasm/code-space-access.h"
#include "src/wasm/compilation-environment-inl.h"
#include "src/wasm/jump-table-assembler.h"
#include "src/wasm/module-decoder.h"
#include "src/wasm/pgo.h"
#include "src/wasm/std-object-sizes.h"
#include "src/wasm/streaming-decoder.h"
#include "src/wasm/wasm-code-manager.h"
#include "src/wasm/wasm-code-pointer-table-inl.h"
#include "src/wasm/wasm-engine.h"
#include "src/wasm/wasm-feature-flags.h"
#include "src/wasm/wasm-import-wrapper-cache.h"
#include "src/wasm/wasm-js.h"
#include "src/wasm/wasm-limits.h"
#include "src/wasm/wasm-objects-inl.h"
#include "src/wasm/wasm-result.h"
#include "src/wasm/wasm-serialization.h"

#define TRACE_COMPILE(...)                                 \
  do {                                                     \
    if (v8_flags.trace_wasm_compiler) PrintF(__VA_ARGS__); \
  } while (false)

#define TRACE_STREAMING(...)                                \
  do {                                                      \
    if (v8_flags.trace_wasm_streaming) PrintF(__VA_ARGS__); \
  } while (false)

#define TRACE_LAZY(...)                                            \
  do {                                                             \
    if (v8_flags.trace_wasm_lazy_compilation) PrintF(__VA_ARGS__); \
  } while (false)

namespace v8::internal::wasm {

namespace {

enum class CompileStrategy : uint8_t {
  // Compiles functions on first use. In this case, execution will block until
  // the function's baseline is reached and top tier compilation starts in
  // background (if applicable).
  // Lazy compilation can help to reduce startup time and code size at the risk
  // of blocking execution.
  kLazy,
  // Compiles baseline ahead of execution and starts top tier compilation in
  // background (if applicable).
  kEager,
  // Marker for default strategy.
  kDefault = kEager,
};

class CompilationStateImpl;
class CompilationUnitBuilder;

class V8_NODISCARD BackgroundCompileScope {
 public:
  explicit BackgroundCompileScope(std::weak_ptr<NativeModule> native_module)
      : native_module_(native_module.lock()) {}

  NativeModule* native_module() const {
    DCHECK(native_module_);
    return native_module_.get();
  }
  inline CompilationStateImpl* compilation_state() const;

  bool cancelled() const;

 private:
  // Keep the native module alive while in this scope.
  std::shared_ptr<NativeModule> native_module_;
};

enum CompilationTier { kBaseline = 0, kTopTier = 1, kNumTiers = kTopTier + 1 };

// A set of work-stealing queues (vectors of units). Each background compile
// task owns one of the queues and steals from all others once its own queue
// runs empty.
class CompilationUnitQueues {
 public:
  // Public API for QueueImpl.
  struct Queue {
    bool ShouldPublish(int num_processed_units) const;
  };

  explicit CompilationUnitQueues(int num_imported_functions,
                                 int num_declared_functions)
      : num_imported_functions_(num_imported_functions),
        num_declared_functions_(num_declared_functions) {
    // Add one first queue, to add units to.
    queues_.emplace_back(std::make_unique<QueueImpl>(0));

#if !defined(__cpp_lib_atomic_value_initialization) || \
    __cpp_lib_atomic_value_initialization < 201911L
    for (auto& atomic_counter : num_units_) {
      std::atomic_init(&atomic_counter, size_t{0});
    }
#endif

    top_tier_compiled_ =
        std::make_unique<std::atomic<bool>[]>(num_declared_functions);

#if !defined(__cpp_lib_atomic_value_initialization) || \
    __cpp_lib_atomic_value_initialization < 201911L
    for (int i = 0; i < num_declared_functions; i++) {
      std::atomic_init(&top_tier_compiled_.get()[i], false);
    }
#endif
  }

  Queue* GetQueueForTask(int task_id) {
    int required_queues = task_id + 1;
    {
      base::MutexGuard queues_guard{&queues_mutex_};
      if (V8_LIKELY(static_cast<int>(queues_.size()) >= required_queues)) {
        return queues_[task_id].get();
      }
    }

    // Otherwise increase the number of queues.
    base::MutexGuard queues_guard{&queues_mutex_};
    int num_queues = static_cast<int>(queues_.size());
    while (num_queues < required_queues) {
      int steal_from = num_queues + 1;
      queues_.emplace_back(std::make_unique<QueueImpl>(steal_from));
      ++num_queues;
    }

    // Update the {publish_limit}s of all queues.

    // We want background threads to publish regularly (to avoid contention when
    // they are all publishing at the end). On the other side, each publishing
    // has some overhead (part of it for synchronizing between threads), so it
    // should not happen *too* often. Thus aim for 4-8 publishes per thread, but
    // distribute it such that publishing is likely to happen at different
    // times.
    int units_per_thread = num_declared_functions_ / num_queues;
    int min = std::max(10, units_per_thread / 8);
    int queue_id = 0;
    for (auto& queue : queues_) {
      // Set a limit between {min} and {2*min}, but not smaller than {10}.
      int limit = min + (min * queue_id / num_queues);
      queue->publish_limit.store(limit, std::memory_order_relaxed);
      ++queue_id;
    }

    return queues_[task_id].get();
  }

  std::optional<WasmCompilationUnit> GetNextUnit(Queue* queue,
                                                 CompilationTier tier) {
    DCHECK_LT(tier, CompilationTier::kNumTiers);
    if (auto unit = GetNextUnitOfTier(queue, tier)) {
      [[maybe_unused]] size_t old_units_count =
          num_units_[tier].fetch_sub(1, std::memory_order_relaxed);
      DCHECK_LE(1, old_units_count);
      return unit;
    }
    return {};
  }

  void AddUnits(base::Vector<WasmCompilationUnit> baseline_units,
                base::Vector<WasmCompilationUnit> top_tier_units,
                const WasmModule* module) {
    DCHECK_LT(0, baseline_units.size() + top_tier_units.size());
    // Add to the individual queues in a round-robin fashion. No special care is
    // taken to balance them; they will be balanced by work stealing.
    QueueImpl* queue;
    {
      int queue_to_add = next_queue_to_add.load(std::memory_order_relaxed);
      base::MutexGuard queues_guard{&queues_mutex_};
      while (!next_queue_to_add.compare_exchange_weak(
          queue_to_add, next_task_id(queue_to_add, queues_.size()),
          std::memory_order_relaxed)) {
        // Retry with updated {queue_to_add}.
      }
      queue = queues_[queue_to_add].get();
    }

    base::MutexGuard guard(&queue->mutex);
    std::optional<base::MutexGuard> big_units_guard;
    for (auto pair :
         {std::make_pair(CompilationTier::kBaseline, baseline_units),
          std::make_pair(CompilationTier::kTopTier, top_tier_units)}) {
      int tier = pair.first;
      base::Vector<WasmCompilationUnit> units = pair.second;
      if (units.empty()) continue;
      num_units_[tier].fetch_add(units.size(), std::memory_order_relaxed);
      for (WasmCompilationUnit unit : units) {
        size_t func_size = module->functions[unit.func_index()].code.length();
        if (func_size <= kBigUnitsLimit) {
          queue->units[tier].push_back(unit);
        } else {
          if (!big_units_guard) {
            big_units_guard.emplace(&big_units_queue_.mutex);
          }
          big_units_queue_.has_units[tier].store(true,
                                                 std::memory_order_relaxed);
          big_units_queue_.units[tier].emplace(func_size, unit);
        }
      }
    }
  }

  void AddTopTierPriorityUnit(WasmCompilationUnit unit, size_t priority) {
    base::MutexGuard queues_guard{&queues_mutex_};
    // Add to the individual queues in a round-robin fashion. No special care is
    // taken to balance them; they will be balanced by work stealing.
    // Priorities should only be seen as a hint here; without balancing, we
    // might pop a unit with lower priority from one queue while other queues
    // still hold higher-priority units.
    // Since updating priorities in a std::priority_queue is difficult, we just
    // add new units with higher priorities, and use the
    // {CompilationUnitQueues::top_tier_compiled_} array to discard units for
    // functions which are already being compiled.
    int queue_to_add = next_queue_to_add.load(std::memory_order_relaxed);
    while (!next_queue_to_add.compare_exchange_weak(
        queue_to_add, next_task_id(queue_to_add, queues_.size()),
        std::memory_order_relaxed)) {
      // Retry with updated {queue_to_add}.
    }

    {
      auto* queue = queues_[queue_to_add].get();
      base::MutexGuard guard(&queue->mutex);
      queue->top_tier_priority_units.emplace(priority, unit);
      num_priority_units_.fetch_add(1, std::memory_order_relaxed);
      num_units_[CompilationTier::kTopTier].fetch_add(
          1, std::memory_order_relaxed);
    }
  }

  // Get the current number of units in the queue for |tier|. This is only a
  // momentary snapshot, it's not guaranteed that {GetNextUnit} returns a unit
  // if this method returns non-zero.
  size_t GetSizeForTier(CompilationTier tier) const {
    DCHECK_LT(tier, CompilationTier::kNumTiers);
    return num_units_[tier].load(std::memory_order_relaxed);
  }

  void AllowAnotherTopTierJob(uint32_t func_index) {
    top_tier_compiled_[declared_function_index(func_index)].store(
        false, std::memory_order_relaxed);
  }

  size_t EstimateCurrentMemoryConsumption() const;

 private:
  // Functions bigger than {kBigUnitsLimit} will be compiled first, in ascending
  // order of their function body size.
  static constexpr size_t kBigUnitsLimit = 4096;

  struct BigUnit {
    BigUnit(size_t func_size, WasmCompilationUnit unit)
        : func_size{func_size}, unit(unit) {}

    size_t func_size;
    WasmCompilationUnit unit;

    bool operator<(const BigUnit& other) const {
      return func_size < other.func_size;
    }
  };

  struct TopTierPriorityUnit {
    TopTierPriorityUnit(int priority, WasmCompilationUnit unit)
        : priority(priority), unit(unit) {}

    size_t priority;
    WasmCompilationUnit unit;

    bool operator<(const TopTierPriorityUnit& other) const {
      return priority < other.priority;
    }
  };

  struct BigUnitsQueue {
    BigUnitsQueue() {
#if !defined(__cpp_lib_atomic_value_initialization) || \
    __cpp_lib_atomic_value_initialization < 201911L
      for (auto& atomic : has_units) std::atomic_init(&atomic, false);
#endif
    }

    mutable base::Mutex mutex;

    // Can be read concurrently to check whether any elements are in the queue.
    std::atomic<bool> has_units[CompilationTier::kNumTiers];

    // Protected by {mutex}:
    std::priority_queue<BigUnit> units[CompilationTier::kNumTiers];
  };

  struct QueueImpl : public Queue {
    explicit QueueImpl(int next_steal_task_id)
        : next_steal_task_id(next_steal_task_id) {}

    // Number of units after which the task processing this queue should publish
    // compilation results. Updated (reduced, using relaxed ordering) when new
    // queues are allocated. If there is only one thread running, we can delay
    // publishing arbitrarily.
    std::atomic<int> publish_limit{kMaxInt};

    base::Mutex mutex;

    // All fields below are protected by {mutex}.
    std::vector<WasmCompilationUnit> units[CompilationTier::kNumTiers];
    std::priority_queue<TopTierPriorityUnit> top_tier_priority_units;
    int next_steal_task_id;
  };

  int next_task_id(int task_id, size_t num_queues) const {
    int next = task_id + 1;
    return next == static_cast<int>(num_queues) ? 0 : next;
  }

  std::optional<WasmCompilationUnit> GetNextUnitOfTier(Queue* public_queue,
                                                       int tier) {
    QueueImpl* queue = static_cast<QueueImpl*>(public_queue);

    // First check whether there is a priority unit. Execute that first.
    if (tier == CompilationTier::kTopTier) {
      if (auto unit = GetTopTierPriorityUnit(queue)) {
        return unit;
      }
    }

    // Then check whether there is a big unit of that tier.
    if (auto unit = GetBigUnitOfTier(tier)) return unit;

    // Finally check whether our own queue has a unit of the wanted tier. If
    // so, return it, otherwise get the task id to steal from.
    int steal_task_id;
    {
      base::MutexGuard mutex_guard(&queue->mutex);
      if (!queue->units[tier].empty()) {
        auto unit = queue->units[tier].back();
        queue->units[tier].pop_back();
        return unit;
      }
      steal_task_id = queue->next_steal_task_id;
    }

    // Try to steal from all other queues. If this succeeds, return one of the
    // stolen units.
    {
      base::MutexGuard guard{&queues_mutex_};
      for (size_t steal_trials = 0; steal_trials < queues_.size();
           ++steal_trials, ++steal_task_id) {
        if (steal_task_id >= static_cast<int>(queues_.size())) {
          steal_task_id = 0;
        }
        if (auto unit = StealUnitsAndGetFirst(queue, steal_task_id, tier)) {
          return unit;
        }
      }
    }

    // If we reach here, we didn't find any unit of the requested tier.
    return {};
  }

  std::optional<WasmCompilationUnit> GetBigUnitOfTier(int tier) {
    // Fast path without locking.
    if (!big_units_queue_.has_units[tier].load(std::memory_order_relaxed)) {
      return {};
    }
    base::MutexGuard guard(&big_units_queue_.mutex);
    if (big_units_queue_.units[tier].empty()) return {};
    WasmCompilationUnit unit = big_units_queue_.units[tier].top().unit;
    big_units_queue_.units[tier].pop();
    if (big_units_queue_.units[tier].empty()) {
      big_units_queue_.has_units[tier].store(false, std::memory_order_relaxed);
    }
    return unit;
  }

  std::optional<WasmCompilationUnit> GetTopTierPriorityUnit(QueueImpl* queue) {
    // Fast path without locking.
    if (num_priority_units_.load(std::memory_order_relaxed) == 0) {
      return {};
    }

    int steal_task_id;
    {
      base::MutexGuard mutex_guard(&queue->mutex);
      while (!queue->top_tier_priority_units.empty()) {
        auto unit = queue->top_tier_priority_units.top().unit;
        queue->top_tier_priority_units.pop();
        num_priority_units_.fetch_sub(1, std::memory_order_relaxed);

        if (!top_tier_compiled_[declared_function_index(unit.func_index())]
                 .exchange(true, std::memory_order_relaxed)) {
          return unit;
        }
        num_units_[CompilationTier::kTopTier].fetch_sub(
            1, std::memory_order_relaxed);
      }
      steal_task_id = queue->next_steal_task_id;
    }

    // Try to steal from all other queues. If this succeeds, return one of the
    // stolen units.
    {
      base::MutexGuard guard{&queues_mutex_};
      for (size_t steal_trials = 0; steal_trials < queues_.size();
           ++steal_trials, ++steal_task_id) {
        if (steal_task_id >= static_cast<int>(queues_.size())) {
          steal_task_id = 0;
        }
        if (auto unit = StealTopTierPriorityUnit(queue, steal_task_id)) {
          return unit;
        }
      }
    }

    return {};
  }

  // Steal units of {wanted_tier} from {steal_from_task_id} to {queue}. Return
  // first stolen unit (rest put in queue of {task_id}), or {nullopt} if
  // {steal_from_task_id} had no units of {wanted_tier}.
  // Hold a shared lock on {queues_mutex_} when calling this method.
  std::optional<WasmCompilationUnit> StealUnitsAndGetFirst(
      QueueImpl* queue, int steal_from_task_id, int wanted_tier) {
    auto* steal_queue = queues_[steal_from_task_id].get();
    // Cannot steal from own queue.
    if (steal_queue == queue) return {};
    std::vector<WasmCompilationUnit> stolen;
    std::optional<WasmCompilationUnit> returned_unit;
    {
      base::MutexGuard guard(&steal_queue->mutex);
      auto* steal_from_vector = &steal_queue->units[wanted_tier];
      if (steal_from_vector->empty()) return {};
      size_t remaining = steal_from_vector->size() / 2;
      auto steal_begin = steal_from_vector->begin() + remaining;
      returned_unit = *steal_begin;
      stolen.assign(steal_begin + 1, steal_from_vector->end());
      steal_from_vector->erase(steal_begin, steal_from_vector->end());
    }
    base::MutexGuard guard(&queue->mutex);
    auto* target_queue = &queue->units[wanted_tier];
    target_queue->insert(target_queue->end(), stolen.begin(), stolen.end());
    queue->next_steal_task_id = steal_from_task_id + 1;
    return returned_unit;
  }

  // Steal one priority unit from {steal_from_task_id} to {task_id}. Return
  // stolen unit, or {nullopt} if {steal_from_task_id} had no priority units.
  // Hold a shared lock on {queues_mutex_} when calling this method.
  std::optional<WasmCompilationUnit> StealTopTierPriorityUnit(
      QueueImpl* queue, int steal_from_task_id) {
    auto* steal_queue = queues_[steal_from_task_id].get();
    // Cannot steal from own queue.
    if (steal_queue == queue) return {};
    std::optional<WasmCompilationUnit> returned_unit;
    {
      base::MutexGuard guard(&steal_queue->mutex);
      while (true) {
        if (steal_queue->top_tier_priority_units.empty()) return {};

        auto unit = steal_queue->top_tier_priority_units.top().unit;
        steal_queue->top_tier_priority_units.pop();
        num_priority_units_.fetch_sub(1, std::memory_order_relaxed);

        if (!top_tier_compiled_[declared_function_index(unit.func_index())]
                 .exchange(true, std::memory_order_relaxed)) {
          returned_unit = unit;
          break;
        }
        num_units_[CompilationTier::kTopTier].fetch_sub(
            1, std::memory_order_relaxed);
      }
    }
    base::MutexGuard guard(&queue->mutex);
    queue->next_steal_task_id = steal_from_task_id + 1;
    return returned_unit;
  }

  int declared_function_index(int func_index) const {
    DCHECK_LE(num_imported_functions_, func_index);
    DCHECK_LT(func_index, num_imported_functions_ + num_declared_functions_);
    return func_index - num_imported_functions_;
  }

  // {queues_mutex_} protectes {queues_};
  mutable base::Mutex queues_mutex_;
  std::vector<std::unique_ptr<QueueImpl>> queues_;

  const int num_imported_functions_;
  const int num_declared_functions_;

  BigUnitsQueue big_units_queue_;

  std::atomic<size_t> num_units_[CompilationTier::kNumTiers];
  std::atomic<size_t> num_priority_units_{0};
  std::unique_ptr<std::atomic<bool>[]> top_tier_compiled_;
  std::atomic<int> next_queue_to_add{0};
};

size_t CompilationUnitQueues::EstimateCurrentMemoryConsumption() const {
  UPDATE_WHEN_CLASS_CHANGES(CompilationUnitQueues, 176);
  UPDATE_WHEN_CLASS_CHANGES(QueueImpl, 112);
  UPDATE_WHEN_CLASS_CHANGES(BigUnitsQueue, 88);
  // Not including sizeof(CompilationUnitQueues) because that's included in
  // sizeof(CompilationStateImpl).
  size_t result = 0;
  {
    base::MutexGuard mutex_guard(&queues_mutex_);
    result += ContentSize(queues_) + queues_.size() * sizeof(QueueImpl);
    for (const auto& q : queues_) {
      base::MutexGuard guard(&q->mutex);
      result += ContentSize(*q->units);
      result += q->top_tier_priority_units.size() * sizeof(TopTierPriorityUnit);
    }
  }
  {
    base::MutexGuard lock(&big_units_queue_.mutex);
    result += big_units_queue_.units[0].size() * sizeof(BigUnit);
    result += big_units_queue_.units[1].size() * sizeof(BigUnit);
  }
  // For {top_tier_compiled_}.
  result += sizeof(std::atomic<bool>) * num_declared_functions_;
  return result;
}

bool CompilationUnitQueues::Queue::ShouldPublish(
    int num_processed_units) const {
  auto* queue = static_cast<const QueueImpl*>(this);
  return num_processed_units >=
         queue->publish_limit.load(std::memory_order_relaxed);
}

// The {CompilationStateImpl} keeps track of the compilation state of the
// owning NativeModule, i.e. which functions are left to be compiled.
// It contains a task manager to allow parallel and asynchronous background
// compilation of functions.
// Its public interface {CompilationState} lives in compilation-environment.h.
class CompilationStateImpl {
 public:
  CompilationStateImpl(const std::shared_ptr<NativeModule>& native_module,
                       WasmDetectedFeatures detected_features);
  ~CompilationStateImpl() {
    if (baseline_compile_job_->IsValid()) {
      baseline_compile_job_->CancelAndDetach();
    }
    if (top_tier_compile_job_->IsValid()) {
      top_tier_compile_job_->CancelAndDetach();
    }
  }

  // Call right after the constructor, after the {compilation_state_} field in
  // the {NativeModule} has been initialized.
  void InitCompileJob();

  // {kCancelUnconditionally}: Cancel all compilation.
  // {kCancelInitialCompilation}: Cancel all compilation if initial (baseline)
  // compilation is not finished yet.
  enum CancellationPolicy { kCancelUnconditionally, kCancelInitialCompilation };
  void CancelCompilation(CancellationPolicy);

  bool cancelled() const;

  // Apply eager tier-up to the initial compilation progress, updating all
  // internal fields accordingly.
  void ApplyEagerTierUpToInitialProgress(size_t hint_idx);

  // Apply a compilation priority hint to initial compilation progress,
  // updating all internal fields accordingly.
  void ApplyCompilationPriorityToInitialProgress(size_t hint_idx,
                                                 CompilationPriority priority);

  // Use PGO information to choose a better initial compilation progress
  // (tiering decisions).
  void ApplyPgoInfoToInitialProgress(ProfileInformation* pgo_info);

  // Apply PGO information to a fully initialized compilation state. Also
  // trigger compilation as needed.
  void ApplyPgoInfoLate(ProfileInformation* pgo_info);

  // Initialize compilation progress. Set compilation tiers to expect for
  // baseline and top tier compilation. Must be set before
  // {CommitCompilationUnits} is invoked which triggers background compilation.
  void InitializeCompilationProgress(ProfileInformation* pgo_info);

  void InitializeCompilationProgressAfterDeserialization(
      base::Vector<const int> lazy_functions,
      base::Vector<const int> eager_functions);

  // Initializes compilation units based on the information encoded in the
  // {compilation_progress_}.
  void InitializeCompilationUnits(
      std::unique_ptr<CompilationUnitBuilder> builder);

  // Adds compilation units for another function to the
  // {CompilationUnitBuilder}. This function is the streaming compilation
  // equivalent to {InitializeCompilationUnits}.
  void InitializeCompilationUnitForSingleFunction(
      CompilationUnitBuilder* builder, int func_index);

  // Add the callback to be called on compilation events. Needs to be
  // set before {CommitCompilationUnits} is run to ensure that it receives all
  // events. The callback object must support being deleted from any thread.
  void AddCallback(std::unique_ptr<CompilationEventCallback> callback);

  // Inserts new functions to compile and kicks off compilation.
  void CommitCompilationUnits(base::Vector<WasmCompilationUnit> baseline_units,
                              base::Vector<WasmCompilationUnit> top_tier_units);
  void CommitTopTierCompilationUnit(WasmCompilationUnit);
  void AddTopTierPriorityCompilationUnit(WasmCompilationUnit, size_t);

  CompilationUnitQueues::Queue* GetQueueForCompileTask(int task_id);

  std::optional<WasmCompilationUnit> GetNextCompilationUnit(
      CompilationUnitQueues::Queue*, CompilationTier tier);

  void OnFinishedUnits(base::Vector<WasmCode*>);

  void OnCompilationStopped(WasmDetectedFeatures detected);
  void SchedulePublishCompilationResults(
      std::vector<UnpublishedWasmCode> unpublished_code, CompilationTier tier);

  WasmDetectedFeatures detected_features() const {
    return detected_features_.load(std::memory_order_relaxed);
  }

  // Update the set of detected features; returns all features that were not
  // detected before.
  V8_WARN_UNUSED_RESULT WasmDetectedFeatures
      UpdateDetectedFeatures(WasmDetectedFeatures);

  size_t NumOutstandingCompilations(CompilationTier tier) const;

  void SetError();

  void WaitForBaselineCompileJob();

  void TierUpAllFunctions();

  void AllowAnotherTopTierJob(uint32_t func_index) {
    compilation_unit_queues_.AllowAnotherTopTierJob(func_index);
    // Reset the stored priority; otherwise triggers might be ignored if the
    // priority is not bumped to the next power of two.
    TypeFeedbackStorage* feedback = &native_module_->module()->type_feedback;
    base::MutexGuard mutex_guard(&feedback->mutex);
    feedback->feedback_for_function[func_index].tierup_priority = 0;
  }

  void AllowAnotherTopTierJobForAllFunctions() {
    const WasmModule* module = native_module_->module();
    uint32_t fn_start = module->num_imported_functions;
    uint32_t fn_end = fn_start + module->num_declared_functions;
    base::MutexGuard mutex_guard(&module->type_feedback.mutex);
    std::unordered_map<uint32_t, FunctionTypeFeedback>& feedback_map =
        module->type_feedback.feedback_for_function;
    for (uint32_t i = fn_start; i < fn_end; i++) {
      compilation_unit_queues_.AllowAnotherTopTierJob(i);
      // Reset the stored priority; otherwise triggers might be ignored if the
      // priority is not bumped to the next power of two.
      if (auto it = feedback_map.find(i); it != feedback_map.end()) {
        it->second.tierup_priority = 0;
      }
    }
  }

  bool failed() const {
    return compile_failed_.load(std::memory_order_relaxed);
  }

  bool baseline_compilation_finished() const {
    base::MutexGuard guard(&callbacks_mutex_);
    return outstanding_baseline_units_ == 0;
  }

  void SetWireBytesStorage(
      std::shared_ptr<WireBytesStorage> wire_bytes_storage) {
    base::MutexGuard guard(&mutex_);
    wire_bytes_storage_ = std::move(wire_bytes_storage);
  }

  std::shared_ptr<WireBytesStorage> GetWireBytesStorage() const {
    base::MutexGuard guard(&mutex_);
    DCHECK_NOT_NULL(wire_bytes_storage_);
    return wire_bytes_storage_;
  }

  void set_compilation_id(int compilation_id) {
    DCHECK_EQ(compilation_id_, kInvalidCompilationID);
    compilation_id_ = compilation_id;
  }

  size_t EstimateCurrentMemoryConsumption() const;

  // Called from the delayed task to trigger caching if the timeout
  // (--wasm-caching-timeout-ms) has passed since the last top-tier compilation.
  // This either triggers caching or re-schedules the task if more code has
  // been compiled to the top tier in the meantime.
  void TriggerCachingAfterTimeout();

  std::vector<WasmCode*> PublishCode(base::Vector<UnpublishedWasmCode> codes);

 private:
  // Trigger callbacks according to the internal counters below
  // (outstanding_...).
  // Hold the {callbacks_mutex_} when calling this method.
  void TriggerOutstandingCallbacks();
  // Trigger an exact set of callbacks. Hold the {callbacks_mutex_} when calling
  // this method.
  void TriggerCallbacks(base::EnumSet<CompilationEvent>);

  void PublishCompilationResults(
      std::vector<UnpublishedWasmCode> unpublished_code);

  NativeModule* const native_module_;
  std::weak_ptr<NativeModule> const native_module_weak_;

  // Compilation error, atomically updated. This flag can be updated and read
  // using relaxed semantics.
  std::atomic<bool> compile_failed_{false};

  // True if compilation was cancelled and worker threads should return. This
  // flag can be updated and read using relaxed semantics.
  std::atomic<bool> compile_cancelled_{false};

  CompilationUnitQueues compilation_unit_queues_;

  // This mutex protects all information of this {CompilationStateImpl} which is
  // being accessed concurrently.
  mutable base::Mutex mutex_;

  // The compile job handles, initialized right after construction of
  // {CompilationStateImpl}.
  std::unique_ptr<JobHandle> baseline_compile_job_;
  std::unique_ptr<JobHandle> top_tier_compile_job_;

  // The compilation id to identify trace events linked to this compilation.
  static constexpr int kInvalidCompilationID = -1;
  int compilation_id_ = kInvalidCompilationID;

  // Features detected to be used in this module. Features can be detected
  // as a module is being compiled.
  std::atomic<WasmDetectedFeatures> detected_features_;

  //////////////////////////////////////////////////////////////////////////////
  // Protected by {mutex_}:

  // Abstraction over the storage of the wire bytes. Held in a shared_ptr so
  // that background compilation jobs can keep the storage alive while
  // compiling.
  std::shared_ptr<WireBytesStorage> wire_bytes_storage_;

  // End of fields protected by {mutex_}.
  //////////////////////////////////////////////////////////////////////////////

  // This mutex protects the callbacks vector, and the counters used to
  // determine which callbacks to call. The counters plus the callbacks
  // themselves need to be synchronized to ensure correct order of events.
  mutable base::Mutex callbacks_mutex_;

  //////////////////////////////////////////////////////////////////////////////
  // Protected by {callbacks_mutex_}:

  // Callbacks to be called on compilation events.
  std::vector<std::unique_ptr<CompilationEventCallback>> callbacks_;

  // Events that already happened.
  base::EnumSet<CompilationEvent> finished_events_;

  int outstanding_baseline_units_ = 0;

  // The amount of generated top tier code since the last
  // {kFinishedCompilationChunk} event.
  size_t bytes_since_last_chunk_ = 0;

  // One byte per declared function, see bitfields defined below.
  // This vector is initialized once to the right size, updates are *usually*
  // protected by the {callbacks_mutes_}, with exceptions during
  // initialization (see comment in
  // {CompilationStateImpl::InitializeCompilationUnitForSingleFunction}).
  base::OwnedVector<uint8_t> compilation_progress_;

  // The timestamp of the last top-tier compilation.
  // This field is updated on every publishing of top-tier code, and is reset
  // once caching is triggered. Hence it also informs whether a caching task is
  // currently being scheduled (whenever this is set).
  base::TimeTicks last_top_tier_compilation_timestamp_;

  // End of fields protected by {callbacks_mutex_}.
  //////////////////////////////////////////////////////////////////////////////

  struct PublishState {
    // {mutex_} protects {publish_queue_} and {publisher_running_}.
    base::Mutex mutex_;
    std::vector<UnpublishedWasmCode> publish_queue_;
    bool publisher_running_ = false;
  };
  PublishState publish_state_[CompilationTier::kNumTiers];

  // Encoding of fields in the {compilation_progress_} vector.
  using RequiredBaselineTierField = base::BitField8<ExecutionTier, 0, 2>;
  using RequiredTopTierField = base::BitField8<ExecutionTier, 2, 2>;
  using ReachedTierField = base::BitField8<ExecutionTier, 4, 2>;
};

CompilationStateImpl* Impl(CompilationState* compilation_state) {
  return reinterpret_cast<CompilationStateImpl*>(compilation_state);
}
const CompilationStateImpl* Impl(const CompilationState* compilation_state) {
  return reinterpret_cast<const CompilationStateImpl*>(compilation_state);
}

CompilationStateImpl* BackgroundCompileScope::compilation_state() const {
  DCHECK(native_module_);
  return Impl(native_module_->compilation_state());
}

size_t CompilationStateImpl::EstimateCurrentMemoryConsumption() const {
  UPDATE_WHEN_CLASS_CHANGES(CompilationStateImpl, 448);
  size_t result = sizeof(CompilationStateImpl);

  {
    base::MutexGuard guard{&mutex_};
    result += compilation_unit_queues_.EstimateCurrentMemoryConsumption();
  }

  // To read the size of {callbacks_} and {compilation_progress_}, we'd
  // need to acquire the {callbacks_mutex_}, which can cause deadlocks
  // when that mutex is already held elsewhere and another thread calls
  // into this function. So we rely on heuristics and informed guesses
  // instead: {compilation_progress_} contains an entry for every declared
  // function in the module...
  result += sizeof(uint8_t) * native_module_->module()->num_declared_functions;
  // ...and there are typically no more than a handful of {callbacks_}.
  constexpr size_t kAssumedNumberOfCallbacks = 4;
  constexpr size_t size_of_vector =
      kAssumedNumberOfCallbacks *
      sizeof(std::unique_ptr<CompilationEventCallback>);
  // Concrete subclasses of CompilationEventCallback will be bigger, but we
  // can't know that here.
  constexpr size_t size_of_payload =
      kAssumedNumberOfCallbacks * sizeof(CompilationEventCallback);
  result += size_of_vector + size_of_payload;

  if (v8_flags.trace_wasm_offheap_memory) {
    PrintF("CompilationStateImpl: %zu\n", result);
  }
  return result;
}

bool BackgroundCompileScope::cancelled() const {
  return native_module_ == nullptr ||
         Impl(native_module_->compilation_state())->cancelled();
}

}  // namespace

//////////////////////////////////////////////////////
// PIMPL implementation of {CompilationState}.

CompilationState::~CompilationState() { Impl(this)->~CompilationStateImpl(); }

void CompilationState::InitCompileJob() { Impl(this)->InitCompileJob(); }

void CompilationState::CancelCompilation() {
  Impl(this)->CancelCompilation(CompilationStateImpl::kCancelUnconditionally);
}

void CompilationState::CancelInitialCompilation() {
  Impl(this)->CancelCompilation(
      CompilationStateImpl::kCancelInitialCompilation);
}

void CompilationState::SetError() { Impl(this)->SetError(); }

void CompilationState::SetWireBytesStorage(
    std::shared_ptr<WireBytesStorage> wire_bytes_storage) {
  Impl(this)->SetWireBytesStorage(std::move(wire_bytes_storage));
}

std::shared_ptr<WireBytesStorage> CompilationState::GetWireBytesStorage()
    const {
  return Impl(this)->GetWireBytesStorage();
}

void CompilationState::AddCallback(
    std::unique_ptr<CompilationEventCallback> callback) {
  return Impl(this)->AddCallback(std::move(callback));
}

void CompilationState::TierUpAllFunctions() {
  Impl(this)->TierUpAllFunctions();
}

void CompilationState::AllowAnotherTopTierJob(uint32_t func_index) {
  Impl(this)->AllowAnotherTopTierJob(func_index);
}

void CompilationState::AllowAnotherTopTierJobForAllFunctions() {
  Impl(this)->AllowAnotherTopTierJobForAllFunctions();
}

void CompilationState::InitializeAfterDeserialization(
    base::Vector<const int> lazy_functions,
    base::Vector<const int> eager_functions) {
  Impl(this)->InitializeCompilationProgressAfterDeserialization(
      lazy_functions, eager_functions);
}

bool CompilationState::failed() const { return Impl(this)->failed(); }

bool CompilationState::baseline_compilation_finished() const {
  return Impl(this)->baseline_compilation_finished();
}

void CompilationState::set_compilation_id(int compilation_id) {
  Impl(this)->set_compilation_id(compilation_id);
}

size_t CompilationState::EstimateCurrentMemoryConsumption() const {
  return Impl(this)->EstimateCurrentMemoryConsumption();
}

std::vector<WasmCode*> CompilationState::PublishCode(
    base::Vector<UnpublishedWasmCode> unpublished_code) {
  return Impl(this)->PublishCode(unpublished_code);
}

// static
std::unique_ptr<CompilationState> CompilationState::New(
    const std::shared_ptr<NativeModule>& native_module,
    WasmDetectedFeatures detected_features) {
  return std::unique_ptr<CompilationState>(reinterpret_cast<CompilationState*>(
      new CompilationStateImpl(native_module, detected_features)));
}

WasmDetectedFeatures CompilationState::detected_features() const {
  return Impl(this)->detected_features();
}

WasmDetectedFeatures CompilationState::UpdateDetectedFeatures(
    WasmDetectedFeatures detected_features) {
  return Impl(this)->UpdateDetectedFeatures(detected_features);
}

// End of PIMPL implementation of {CompilationState}.
//////////////////////////////////////////////////////

namespace {

struct ExecutionTierPair {
  ExecutionTier baseline_tier;
  ExecutionTier top_tier;
};

// Pass the debug state as a separate parameter to avoid data races: the debug
// state may change between its use here and its use at the call site. To have
// a consistent view on the debug state, the caller reads the debug state once
// and then passes it to this function.
ExecutionTierPair GetDefaultTiersPerModule(NativeModule* native_module,
                                           DebugState is_in_debug_state,
                                           bool lazy_module) {
  const WasmModule* module = native_module->module();
  if (lazy_module) {
    return {ExecutionTier::kNone, ExecutionTier::kNone};
  }
  if (is_asmjs_module(module)) {
    DCHECK(!is_in_debug_state);
    return {ExecutionTier::kTurbofan, ExecutionTier::kTurbofan};
  }
  if (is_in_debug_state) {
    return {ExecutionTier::kLiftoff, ExecutionTier::kLiftoff};
  }
  ExecutionTier baseline_tier =
      v8_flags.liftoff ? ExecutionTier::kLiftoff : ExecutionTier::kTurbofan;
  bool eager_tier_up = !v8_flags.wasm_dynamic_tiering && v8_flags.wasm_tier_up;
  ExecutionTier top_tier =
      eager_tier_up ? ExecutionTier::kTurbofan : baseline_tier;
  return {baseline_tier, top_tier};
}

ExecutionTierPair GetLazyCompilationTiers(NativeModule* native_module,
                                          uint32_t func_index,
                                          DebugState is_in_debug_state) {
  // For lazy compilation, get the tiers we would use if lazy compilation is
  // disabled.
  constexpr bool kNotLazy = false;
  ExecutionTierPair tiers =
      GetDefaultTiersPerModule(native_module, is_in_debug_state, kNotLazy);
  // If we are in debug mode, we ignore the tier-up filter.
  if (is_in_debug_state) return tiers;

  if (native_module->enabled_features().has_compilation_hints()) {
    if (auto priority =
            native_module->module()->GetCompilationPriority(func_index)) {
      DCHECK_LE(priority->optimization_priority,
                kOptimizationPriorityExecutedOnceSentinel);
      if (priority->optimization_priority ==
          kOptimizationPriorityExecutedOnceSentinel) {
        // In this case, the function is only executed once. We do not want to
        // tier it up to Turbofan.
        tiers.top_tier = ExecutionTier::kNone;
      }
    }
  }

  if (V8_UNLIKELY(v8_flags.wasm_tier_up_filter >= 0 &&
                  func_index !=
                      static_cast<uint32_t>(v8_flags.wasm_tier_up_filter) &&
                  tiers.top_tier != ExecutionTier::kNone)) {
    tiers.top_tier = tiers.baseline_tier;
  }

  // Correct top tier if necessary.
  static_assert(ExecutionTier::kLiftoff < ExecutionTier::kTurbofan,
                "Assume an order on execution tiers");
  if (tiers.baseline_tier > tiers.top_tier) {
    tiers.top_tier = tiers.baseline_tier;
  }
  return tiers;
}

// The {CompilationUnitBuilder} builds compilation units and stores them in an
// internal buffer. The buffer is moved into the working queue of the
// {CompilationStateImpl} when {Commit} is called.
class CompilationUnitBuilder {
 public:
  explicit CompilationUnitBuilder(NativeModule* native_module)
      : native_module_(native_module) {}

  void AddBaselineUnit(int func_index, ExecutionTier tier) {
    baseline_units_.emplace_back(func_index, tier, kNotForDebugging);
  }

  void AddTopTierUnit(int func_index, ExecutionTier tier) {
    tiering_units_.emplace_back(func_index, tier, kNotForDebugging);
  }

  void Commit() {
    if (baseline_units_.empty() && tiering_units_.empty()) return;
    compilation_state()->CommitCompilationUnits(base::VectorOf(baseline_units_),
                                                base::VectorOf(tiering_units_));
    Clear();
  }

  void Clear() {
    baseline_units_.clear();
    tiering_units_.clear();
  }

  const WasmModule* module() { return native_module_->module(); }

 private:
  CompilationStateImpl* compilation_state() const {
    return Impl(native_module_->compilation_state());
  }

  NativeModule* const native_module_;
  std::vector<WasmCompilationUnit> baseline_units_;
  std::vector<WasmCompilationUnit> tiering_units_;
};

DecodeResult ValidateSingleFunction(Zone* zone, const WasmModule* module,
                                    int func_index,
                                    base::Vector<const uint8_t> code,
                                    WasmEnabledFeatures enabled_features,
                                    WasmDetectedFeatures* detected_features) {
  // Sometimes functions get validated unpredictably in the background, for
  // debugging or when inlining one function into another. We check here if that
  // is the case, and exit early if so.
  if (module->function_was_validated(func_index)) return {};
  const WasmFunction* func = &module->functions[func_index];
  bool is_shared = module->type(func->sig_index).is_shared;
  FunctionBody body{func->sig, func->code.offset(), code.begin(), code.end(),
                    is_shared};
  DecodeResult result = ValidateFunctionBody(zone, enabled_features, module,
                                             detected_features, body);
  if (result.ok()) module->set_function_validated(func_index);
  return result;
}

enum OnlyLazyFunctions : bool {
  kAllFunctions = false,
  kOnlyLazyFunctions = true,
};

bool IsLazyModule(const WasmModule* module) {
  return v8_flags.wasm_lazy_compilation ||
         (v8_flags.asm_wasm_lazy_compilation && is_asmjs_module(module));
}

class CompileLazyTimingScope {
 public:
  CompileLazyTimingScope(Counters* counters, NativeModule* native_module)
      : counters_(counters), native_module_(native_module) {
    timer_.Start();
  }

  ~CompileLazyTimingScope() {
    base::TimeDelta elapsed = timer_.Elapsed();
    native_module_->AddLazyCompilationTimeSample(elapsed.InMicroseconds());
    counters_->wasm_lazy_compile_time()->AddTimedSample(elapsed);
  }

 private:
  Counters* counters_;
  NativeModule* native_module_;
  base::ElapsedTimer timer_;
};

}  // namespace

bool CompileLazy(Isolate* isolate, NativeModule* native_module,
                 int func_index) {
  DisallowGarbageCollection no_gc;
  Counters* counters = isolate->counters();

  // Put the timer scope around everything, including the {CodeSpaceWriteScope}
  // and its destruction, to measure complete overhead (apart from the runtime
  // function itself, which has constant overhead).
  std::optional<CompileLazyTimingScope> lazy_compile_time_scope;
  if (base::TimeTicks::IsHighResolution()) {
    lazy_compile_time_scope.emplace(counters, native_module);
  }

  DCHECK(!native_module->lazy_compile_frozen());

  TRACE_LAZY("Compiling wasm-function#%d.\n", func_index);

  CompilationStateImpl* compilation_state =
      Impl(native_module->compilation_state());
  DebugState is_in_debug_state = native_module->IsInDebugState();
  ExecutionTierPair tiers =
      GetLazyCompilationTiers(native_module, func_index, is_in_debug_state);

  DCHECK_LE(native_module->num_imported_functions(), func_index);
  DCHECK_LT(func_index, native_module->num_functions());
  WasmCompilationUnit baseline_unit{
      func_index, tiers.baseline_tier,
      is_in_debug_state ? kForDebugging : kNotForDebugging};
  CompilationEnv env = CompilationEnv::ForModule(native_module);
  WasmDetectedFeatures detected_features;
  WasmCompilationResult result = baseline_unit.ExecuteCompilation(
      &env, compilation_state->GetWireBytesStorage().get(),
      native_module->counter_updates(), &detected_features);
  compilation_state->OnCompilationStopped(detected_features);

  // During lazy compilation, we can only get compilation errors when
  // {--wasm-lazy-validation} is enabled. Otherwise, the module was fully
  // verified before starting its execution.
  CHECK_IMPLIES(result.failed(), v8_flags.wasm_lazy_validation);
  if (result.failed()) {
    return false;
  }

  WasmCodeRefScope code_ref_scope;
  WasmCode* code =
      native_module->PublishCode(native_module->AddCompiledCode(result));
  DCHECK_EQ(func_index, code->index());

  if (V8_UNLIKELY(native_module->log_code())) {
    GetWasmEngine()->LogCode(base::VectorOf(&code, 1));
    // Log the code immediately in the current isolate.
    GetWasmEngine()->LogOutstandingCodesForIsolate(isolate);
  }

  counters->wasm_lazily_compiled_functions()->Increment();

  const WasmModule* module = native_module->module();
  const bool lazy_module = IsLazyModule(module);
  if (lazy_module && tiers.baseline_tier < tiers.top_tier) {
    WasmCompilationUnit tiering_unit{func_index, tiers.top_tier,
                                     kNotForDebugging};
    compilation_state->CommitTopTierCompilationUnit(tiering_unit);
  }
  return true;
}

void ThrowLazyCompilationError(Isolate* isolate,
                               const NativeModule* native_module,
                               int func_index) {
  const WasmModule* module = native_module->module();

  CompilationStateImpl* compilation_state =
      Impl(native_module->compilation_state());
  const WasmFunction* func = &module->functions[func_index];
  base::Vector<const uint8_t> code =
      compilation_state->GetWireBytesStorage()->GetCode(func->code);

  auto enabled_features = native_module->enabled_features();
  // This path is unlikely, so the overhead for creating an extra Zone is
  // not important.
  Zone validation_zone{GetWasmEngine()->allocator(), ZONE_NAME};
  WasmDetectedFeatures unused_detected_features;
  DecodeResult decode_result =
      ValidateSingleFunction(&validation_zone, module, func_index, code,
                             enabled_features, &unused_detected_features);

  CHECK(decode_result.failed());
  wasm::ErrorThrower thrower(isolate, nullptr);
  thrower.CompileFailed(GetWasmErrorWithName(native_module->wire_bytes(),
                                             func_index, module,
                                             std::move(decode_result).error()));
}

bool IsCrossInstanceCall(Tagged<Object> obj, Isolate* const isolate) {
  return obj == ReadOnlyRoots{isolate}.wasm_cross_instance_call_symbol();
}

class FeedbackMaker {
 public:
  FeedbackMaker(Isolate* const isolate,
                Tagged<WasmTrustedInstanceData> trusted_instance_data,
                int func_index, int num_calls)
      : isolate_(isolate),
        instance_data_(trusted_instance_data),
        result_(
            base::OwnedVector<CallSiteFeedback>::NewForOverwrite(num_calls)),
        num_imported_functions_(static_cast<int>(
            trusted_instance_data->module()->num_imported_functions)),
        func_index_(func_index) {}

  void AddCallRefCandidate(Tagged<WasmFuncRef> funcref, int count) {
    Tagged<WasmInternalFunction> internal_function =
        funcref->internal(isolate_);
    // Discard cross-instance calls, as we can only inline same-instance code.
    if (internal_function->implicit_arg() != instance_data_) {
      has_non_inlineable_targets_ = true;
      return;
    }
    // Discard imports for now.
    if (internal_function->function_index() < num_imported_functions_) {
      has_non_inlineable_targets_ = true;
      return;
    }
    AddCall(internal_function->function_index(), count);
  }

  void AddCallIndirectCandidate(Tagged<Object> target_truncated_obj,
                                int count) {
    // Discard cross-instance calls, as we can only inline same-instance code.
    if (IsCrossInstanceCall(target_truncated_obj, isolate_)) {
      has_non_inlineable_targets_ = true;
      return;
    }
    Tagged<Smi> target_truncated_smi = Cast<Smi>(target_truncated_obj);

    // We need to map a truncated call target back to a function index.
    // Generally there may be multiple jump tables if code spaces are far apart
    // (to ensure that direct calls can always use a near call to the closest
    // jump table).
    // However, here we are always handling call targets that are originally
    // from the `WasmDispatchTable`, whose entries are always targets pointing
    // into the main jump table, so we only need to check against that.

    // Feedback data is untrusted. An invalid handle could lead to an OOB read
    // from the `WasmCodePointerTable`, which either crashes or returns some
    // garbage. Thus `WasmCodeManager::LookupCode` would either return `nullptr`
    // (safe) or an unrelated code object which will be checked for a compatible
    // signature before being inlined.
    // So the CHECK here is mostly just there to silence a false positive report
    // by the sandbox crash filter about an OOB read.
    uint32_t untrusted_code_pointer =
        static_cast<uint32_t>(target_truncated_smi.value());
    CHECK_LT(untrusted_code_pointer, WasmCodePointer::kIndexSpaceSize);
    WasmCodePointer handle = WasmCodePointer{untrusted_code_pointer};
    Address entry = GetProcessWideWasmCodePointerTable()
                        ->GetEntrypointWithoutSignatureCheck(handle);
    wasm::WasmCode* code =
        wasm::GetWasmCodeManager()->LookupCode(nullptr, entry);
    if (!code || code->native_module() != instance_data_->native_module() ||
        code->IsAnonymous()) {
      // Was not in the main table (e.g., because it's an imported function).
      has_non_inlineable_targets_ = true;
      return;
    }
    DCHECK_EQ(code->kind(), WasmCode::Kind::kWasmFunction);
    uint32_t func_idx = code->index();
    AddCall(func_idx, count);
  }

  void AddCall(int target, int count) {
    // If we add too many calls, treat it as megamorphic.
    if (static_cast<size_t>(cache_usage_) == targets_cache_.size() ||
        is_megamorphic_) {
      is_megamorphic_ = true;
      return;
    }
    // Keep the cache sorted (using insertion-sort), highest count first.
    int insertion_index = 0;
    while (insertion_index < cache_usage_ &&
           counts_cache_[insertion_index] >= count) {
      insertion_index++;
    }
    for (int shifted_index = cache_usage_ - 1; shifted_index >= insertion_index;
         shifted_index--) {
      targets_cache_[shifted_index + 1] = targets_cache_[shifted_index];
      counts_cache_[shifted_index + 1] = counts_cache_[shifted_index];
    }
    targets_cache_[insertion_index] = target;
    counts_cache_[insertion_index] = count;
    cache_usage_++;
  }

  bool HasTargetCached(int target) {
    auto end = targets_cache_.begin() + cache_usage_;
    DCHECK_LE(end, targets_cache_.end());
    return std::find(targets_cache_.begin(), end, target) != end;
  }

  void AddResult(CallSiteFeedback feedback) {
    DCHECK_LT(seen_calls_, result_.size());
    result_[seen_calls_] = feedback;
    ++seen_calls_;
  }

  void FinalizeCall() {
    if (is_megamorphic_) {
      if (v8_flags.trace_wasm_inlining) {
        PrintF("[function %d: call #%d: megamorphic]\n", func_index_,
               seen_calls_);
      }
      AddResult(CallSiteFeedback::CreateMegamorphic());
    } else if (cache_usage_ == 0) {
      AddResult(CallSiteFeedback{});
    } else if (cache_usage_ == 1) {
      if (v8_flags.trace_wasm_inlining) {
        PrintF("[function %d: call #%d inlineable (monomorphic)]\n",
               func_index_, seen_calls_);
      }
      AddResult(CallSiteFeedback{targets_cache_[0], counts_cache_[0]});
    } else {
      if (v8_flags.trace_wasm_inlining) {
        PrintF("[function %d: call #%d inlineable (polymorphic %d)]\n",
               func_index_, seen_calls_, cache_usage_);
      }
      DCHECK_LE(cache_usage_, kMaxPolymorphism);
      CallSiteFeedback::PolymorphicCase* polymorphic =
          new CallSiteFeedback::PolymorphicCase[cache_usage_];
      for (int i = 0; i < cache_usage_; i++) {
        polymorphic[i].function_index = targets_cache_[i];
        polymorphic[i].absolute_call_frequency = counts_cache_[i];
      }
      AddResult(CallSiteFeedback{polymorphic, cache_usage_});
    }
    result_[seen_calls_ - 1].set_has_non_inlineable_targets(
        has_non_inlineable_targets_);
    // TODO(mliedtke): Have a better representation that merges these properties
    // into one object.
    has_non_inlineable_targets_ = false;
    is_megamorphic_ = false;
    cache_usage_ = 0;
  }

  void set_has_non_inlineable_targets() { has_non_inlineable_targets_ = true; }
  void set_megamorphic() { is_megamorphic_ = true; }

  // {GetResult} can only be called on a r-value reference to make it more
  // obvious at call sites that {this} should not be used after this operation.
  base::OwnedVector<CallSiteFeedback> GetResult() && {
    return std::move(result_);
  }

 private:
  Isolate* const isolate_;
  const Tagged<WasmTrustedInstanceData> instance_data_;
  base::OwnedVector<CallSiteFeedback> result_;
  int seen_calls_ = 0;
  const int num_imported_functions_;
  const int func_index_;
  int cache_usage_{0};
  std::array<int, kMaxPolymorphism> targets_cache_;
  std::array<int, kMaxPolymorphism> counts_cache_;
  bool has_non_inlineable_targets_ = false;
  // If we add more call targets than kMaxPolymorphism while processing the
  // feedback, treat it as megamorphic.
  bool is_megamorphic_ = false;
};

TransitiveTypeFeedbackProcessor::TransitiveTypeFeedbackProcessor(
    Isolate* isolate, Tagged<WasmTrustedInstanceData> trusted_instance_data)
    : isolate_(isolate),
      instance_data_(trusted_instance_data),
      module_(trusted_instance_data->module()),
      mutex_guard(&module_->type_feedback.mutex),
      feedback_for_function_(module_->type_feedback.feedback_for_function) {}

void TransitiveTypeFeedbackProcessor::ProcessAll(
    Isolate* isolate, Tagged<WasmTrustedInstanceData> trusted_instance_data) {
  TransitiveTypeFeedbackProcessor ttfp{isolate, trusted_instance_data};
  const WasmModule* module = trusted_instance_data->native_module()->module();
  for (int func_index = module->num_imported_functions;
       func_index < static_cast<int>(module->functions.size()); func_index++) {
    ttfp.queue_.insert(func_index);
  }
  ttfp.ProcessQueue();
}

void TransitiveTypeFeedbackProcessor::ProcessFunction(int func_index) {
  int which_vector = declared_function_index(module_, func_index);
  Tagged<Object> maybe_feedback =
      instance_data_->feedback_vectors()->get(which_vector);
  if (!IsFixedArray(maybe_feedback)) return;
  Tagged<FixedArray> feedback = Cast<FixedArray>(maybe_feedback);
  base::Vector<uint32_t> call_targets =
      module_->type_feedback.feedback_for_function[func_index]
          .call_targets.as_vector();

  // For each entry in {call_targets}, there are two {Object} slots in the
  // {feedback} vector:
  // +--------------------------+-----------------------------+----------------+
  // |        Call Type         |      Feedback: Entry 1      |    Entry 2     |
  // +-------------------------+------------------------------+----------------+
  // | direct                   | Smi(count)                  | Smi(0), unused |
  // +--------------------------+-----------------------------+----------------+
  // | ref, uninitialized       | Smi(0)                      | Smi(0)         |
  // | ref, monomorphic         | WasmFuncRef(target)         | Smi(count>0)   |
  // | ref, polymorphic         | FixedArray                  | Undefined      |
  // | ref, megamorphic         | MegamorphicSymbol           | Undefined      |
  // +--------------------------+-----------------------------+----------------+
  // | indirect, uninitialized  | Smi(0)                      | Smi(0)         |
  // | indirect, monomorphic    | Smi(truncated_target)       | Smi(count>0)   |
  // | indirect, wrong instance | WasmCrossInstanceCallSymbol | Smi(count>0)   |
  // | indirect, polymorphic    | FixedArray                  | Undefined      |
  // | indirect, megamorphic    | MegamorphicSymbol           | Undefined      |
  // +--------------------------+-----------------------------+----------------+
  // The FixedArray entries for the polymorphic cases look like the monomorphic
  // entries in the feedback vector itself, i.e., they can a (truncated) target,
  // or the wrong instance sentinel (for cross-instance call_indirect).
  // See {UpdateCallRefOrIndirectIC} in {wasm.tq} for how this is written.
  // Since this is combining untrusted data ({feedback} vector on the JS heap)
  // with trusted data ({call_targets}), make sure to avoid an OOB access.
  int checked_feedback_length = feedback->length();
  // The first slot is reserved for total invocation count.
  SBXCHECK_EQ(checked_feedback_length,
              call_targets.size() * FeedbackConstants::kSlotsPerInstruction +
                  FeedbackConstants::kHeaderSlots);
  FeedbackMaker fm(isolate_, instance_data_, func_index,
                   (checked_feedback_length - FeedbackConstants::kHeaderSlots) /
                       FeedbackConstants::kSlotsPerInstruction);
  for (int i = FeedbackConstants::kHeaderSlots; i < checked_feedback_length;
       i += FeedbackConstants::kSlotsPerInstruction) {
    uint32_t sentinel_or_target =
        call_targets[(i - FeedbackConstants::kHeaderSlots) /
                     FeedbackConstants::kSlotsPerInstruction];
    Tagged<Object> first_slot = feedback->get(i);
    Tagged<Object> second_slot = feedback->get(i + 1);

    if (sentinel_or_target != FunctionTypeFeedback::kCallRef &&
        sentinel_or_target != FunctionTypeFeedback::kCallIndirect) {
      // Direct call counts.
      int count = Smi::ToInt(first_slot);
      DCHECK_EQ(Smi::ToInt(second_slot), 0);
      // TODO(dlehmann): Currently, TurboFan assumes that we add feedback even
      // if the call count is zero. Once TurboFan is gone, revisit if we can
      // avoid this (similar to how we do for call_ref/call_indirect today).
      fm.AddCall(static_cast<int>(sentinel_or_target), count);
    } else if (IsSmi(second_slot) && Smi::ToInt(second_slot) == 0) {
      // Uninitialized call_ref or call_indirect.
      DCHECK_EQ(Smi::ToInt(first_slot), 0);
      if (v8_flags.trace_wasm_inlining) {
        PrintF("[function %d: call #%d: uninitialized]\n", func_index,
               (i - FeedbackConstants::kHeaderSlots) /
                   FeedbackConstants::kSlotsPerInstruction);
      }
    } else if (IsWasmFuncRef(first_slot)) {
      // Monomorphic call_ref.
      DCHECK_EQ(sentinel_or_target, FunctionTypeFeedback::kCallRef);
      int count = Smi::ToInt(second_slot);
      fm.AddCallRefCandidate(Cast<WasmFuncRef>(first_slot), count);
    } else if (IsSmi(first_slot) || IsCrossInstanceCall(first_slot, isolate_)) {
      // Monomorphic call_indirect.
      DCHECK_EQ(sentinel_or_target, FunctionTypeFeedback::kCallIndirect);
      int count = Smi::ToInt(second_slot);
      fm.AddCallIndirectCandidate(first_slot, count);
    } else if (IsFixedArray(first_slot)) {
      // Polymorphic call_ref or call_indirect.
      Tagged<FixedArray> polymorphic = Cast<FixedArray>(first_slot);
      DCHECK(IsUndefined(second_slot));
      int checked_polymorphic_length = polymorphic->length();
      SBXCHECK_LE(checked_polymorphic_length,
                  FeedbackConstants::kSlotsPerInstruction * kMaxPolymorphism);
      if (sentinel_or_target == FunctionTypeFeedback::kCallRef) {
        for (int j = 0; j < checked_polymorphic_length;
             j += FeedbackConstants::kSlotsPerInstruction) {
          Tagged<WasmFuncRef> target = Cast<WasmFuncRef>(polymorphic->get(j));
          int count = Smi::ToInt(polymorphic->get(j + 1));
          fm.AddCallRefCandidate(target, count);
        }
      } else {
        DCHECK_EQ(sentinel_or_target, FunctionTypeFeedback::kCallIndirect);
        for (int j = 0; j < checked_polymorphic_length;
             j += FeedbackConstants::kSlotsPerInstruction) {
          Tagged<Object> target = polymorphic->get(j);
          int count = Smi::ToInt(polymorphic->get(j + 1));
          fm.AddCallIndirectCandidate(target, count);
        }
      }
    } else if (first_slot == ReadOnlyRoots{isolate_}.megamorphic_symbol()) {
      DCHECK(IsUndefined(second_slot));
      fm.set_megamorphic();
    } else {
      UNREACHABLE();
    }

    if (v8_flags.wasm_deopt &&
        first_slot != ReadOnlyRoots{isolate_}.megamorphic_symbol()) {
      // If we already had feedback for this call, also add the already existing
      // feedback to prevent deopt loops where two different instantiations
      // (which have their own on-heap feedback vector) to "flip-flop" between
      // their inlining decisions potentially causing deopt loops.
      const base::OwnedVector<CallSiteFeedback>& existing =
          feedback_for_function_[func_index].feedback_vector;
      size_t feedback_index = (i - FeedbackConstants::kHeaderSlots) /
                              FeedbackConstants::kSlotsPerInstruction;
      if (feedback_index < existing.size()) {
        const CallSiteFeedback& old_feedback = existing[feedback_index];
        if (old_feedback.has_non_inlineable_targets()) {
          fm.set_has_non_inlineable_targets();
        }
        if (old_feedback.is_megamorphic()) {
          fm.set_megamorphic();
        }
        for (int j = 0; j < old_feedback.num_cases(); ++j) {
          int old_target_function_index = old_feedback.function_index(j);
          // If the new feedback already contains the target, we do not touch
          // the call count.
          if (!fm.HasTargetCached(old_target_function_index)) {
            fm.AddCall(old_target_function_index, old_feedback.call_count(j));
            // There shouldn't be any imported functions in there as they can't
            // be inlined. If this DCHECK is invalidated,
            // has_non_inlineable_targets_ would need to be updated here to
            // reflect that.
            DCHECK_GE(static_cast<uint32_t>(old_target_function_index),
                      instance_data_->module()->num_imported_functions);
          }
        }
      }
    }

    fm.FinalizeCall();
  }
  base::OwnedVector<CallSiteFeedback> result = std::move(fm).GetResult();
  EnqueueCallees(result.as_vector());
  DCHECK_EQ(result.size(),
            feedback_for_function_[func_index].call_targets.size());
  feedback_for_function_[func_index].feedback_vector = std::move(result);
  feedback_for_function_[func_index].num_invocations =
      Cast<Smi>(feedback->get(0)).value();
}

void TriggerTierUp(Isolate* isolate,
                   Tagged<WasmTrustedInstanceData> trusted_instance_data,
                   int func_index) {
  NativeModule* native_module = trusted_instance_data->native_module();
  CompilationStateImpl* compilation_state =
      Impl(native_module->compilation_state());
  WasmCompilationUnit tiering_unit{func_index, ExecutionTier::kTurbofan,
                                   kNotForDebugging};

  const WasmModule* module = native_module->module();
  int priority;
  {
    base::MutexGuard mutex_guard(&module->type_feedback.mutex);
    int array_index = wasm::declared_function_index(module, func_index);
    trusted_instance_data->tiering_budget_array()[array_index].store(
        v8_flags.wasm_tiering_budget, std::memory_order_relaxed);
    int& stored_priority =
        module->type_feedback.feedback_for_function[func_index].tierup_priority;
    if (stored_priority < kMaxInt) ++stored_priority;
    priority = stored_priority;
  }
  // Only create a compilation unit if this is the first time we detect this
  // function as hot (priority == 1), or if the priority increased
  // significantly. The latter is assumed to be the case if the priority
  // increased at least to four, and is a power of two.
  if (priority == 2 || !base::bits::IsPowerOfTwo(priority)) return;

  // Before adding the tier-up unit or increasing priority, process type
  // feedback for best code generation.
  if (v8_flags.wasm_inlining) {
    // TODO(jkummerow): we could have collisions here if different instances
    // of the same module have collected different feedback. If that ever
    // becomes a problem, figure out a solution.
    TransitiveTypeFeedbackProcessor::Process(isolate, trusted_instance_data,
                                             func_index);
  }

  compilation_state->AddTopTierPriorityCompilationUnit(tiering_unit, priority);
}

void TierUpNowForTesting(Isolate* isolate,
                         Tagged<WasmTrustedInstanceData> trusted_instance_data,
                         int func_index) {
  NativeModule* native_module = trusted_instance_data->native_module();
  if (v8_flags.wasm_inlining) {
    TransitiveTypeFeedbackProcessor::Process(isolate, trusted_instance_data,
                                             func_index);
  }
  wasm::GetWasmEngine()->CompileFunction(native_module, func_index,
                                         wasm::ExecutionTier::kTurbofan);
  CHECK(!native_module->compilation_state()->failed());
}

void TierUpAllForTesting(
    Isolate* isolate, Tagged<WasmTrustedInstanceData> trusted_instance_data) {
  NativeModule* native_module = trusted_instance_data->native_module();
  const WasmModule* mod = native_module->module();
  WasmCodeRefScope code_ref_scope;

  uint32_t start = mod->num_imported_functions;
  uint32_t end = start + mod->num_declared_functions;
  for (uint32_t func_index = start; func_index < end; func_index++) {
    if (!native_module->HasCodeWithTier(func_index, ExecutionTier::kTurbofan)) {
      TierUpNowForTesting(isolate, trusted_instance_data, func_index);
    }
  }
}

void InitializeCompilationForTesting(NativeModule* native_module) {
  Impl(native_module->compilation_state())
      ->InitializeCompilationProgress(nullptr);
}

void PublishDetectedFeatures(WasmDetectedFeatures detected_features,
                             Isolate* isolate, bool is_initial_compilation) {
  using Feature = v8::Isolate::UseCounterFeature;
  static constexpr std::pair<WasmDetectedFeature, Feature> kUseCounters[] = {
      {WasmDetectedFeature::shared_memory, Feature::kWasmSharedMemory},
      {WasmDetectedFeature::reftypes, Feature::kWasmRefTypes},
      {WasmDetectedFeature::simd, Feature::kWasmSimdOpcodes},
      {WasmDetectedFeature::threads, Feature::kWasmThreadOpcodes},
      {WasmDetectedFeature::legacy_eh, Feature::kWasmExceptionHandling},
      {WasmDetectedFeature::memory64, Feature::kWasmMemory64},
      {WasmDetectedFeature::multi_memory, Feature::kWasmMultiMemory},
      {WasmDetectedFeature::gc, Feature::kWasmGC},
      {WasmDetectedFeature::imported_strings, Feature::kWasmImportedStrings},
      {WasmDetectedFeature::imported_strings_utf8,
       Feature::kWasmImportedStringsUtf8},
      {WasmDetectedFeature::return_call, Feature::kWasmReturnCall},
      {WasmDetectedFeature::extended_const, Feature::kWasmExtendedConst},
      {WasmDetectedFeature::relaxed_simd, Feature::kWasmRelaxedSimd},
      {WasmDetectedFeature::type_reflection, Feature::kWasmTypeReflection},
      {WasmDetectedFeature::exnref, Feature::kWasmExnRef},
      {WasmDetectedFeature::typed_funcref, Feature::kWasmTypedFuncRef},
      {WasmDetectedFeature::jspi, Feature::kWasmJavaScriptPromiseIntegration},
      {WasmDetectedFeature::branch_hinting, Feature::kWasmBranchHinting},
      {WasmDetectedFeature::multi_value, Feature::kWasmMultiValue},
      {WasmDetectedFeature::bulk_memory, Feature::kWasmBulkMemory},
      {WasmDetectedFeature::mutable_globals, Feature::kWasmMutableGlobals},
      {WasmDetectedFeature::non_trapping_float_to_int,
       Feature::kWasmNonTrappingFloatToInt},
      {WasmDetectedFeature::sign_extension_ops, Feature::kWasmSignExtensionOps},
      {WasmDetectedFeature::custom_descriptors,
       Feature::kWasmCustomDescriptors},
      {WasmDetectedFeature::rab_integration, Feature::kWasmResizableBuffers},
  };

  // Check that every staging or shipping feature has a use counter as that is
  // the main point of tracking used features.
  auto check_use_counter = [](WasmDetectedFeature feat) constexpr -> bool {
    // Some features intentionally do not have a use counter.
    constexpr WasmDetectedFeature kIntentionallyNoUseCounter[] = {
        // Deprecated / unlikely to ship.
        WasmDetectedFeature::stringref,
        // Growable stacks is not a language feature but an internal
        // optimization.
        // We could detect stack growth events but it's unclear how we would use
        // the data, and it could affect performance.
        WasmDetectedFeature::growable_stacks,
    };
    for (auto no_use_counter_feature : kIntentionallyNoUseCounter) {
      if (feat == no_use_counter_feature) return true;
    }
    for (auto [feature, use_counter] : kUseCounters) {
      if (feat == feature) return true;
    }
    return false;
  };
#define CHECK_USE_COUNTER(feat, ...) \
  static_assert(check_use_counter(WasmDetectedFeature::feat));
  FOREACH_WASM_STAGING_FEATURE_FLAG(CHECK_USE_COUNTER)
  FOREACH_WASM_SHIPPED_FEATURE_FLAG(CHECK_USE_COUNTER)
  FOREACH_WASM_NON_FLAG_FEATURE(CHECK_USE_COUNTER)
#undef CHECK_USE_COUNTER

  static constexpr size_t kMaxFeatures = arraysize(kUseCounters) + 1;
  base::SmallVector<Feature, kMaxFeatures> use_counter_features;
  if (is_initial_compilation) {
    // Always set the WasmModuleCompilation feature as a baseline for the other
    // features. Note that we also track instantiation, but the number of
    // compilations and instantiations are pretty unrelated.
    use_counter_features.push_back(Feature::kWasmModuleCompilation);
  }

  for (auto [wasm_feature, feature] : kUseCounters) {
    if (!detected_features.contains(wasm_feature)) continue;
    use_counter_features.push_back(feature);
  }
  if (use_counter_features.empty()) return;

  isolate->CountUsage(base::VectorOf(use_counter_features));

  // Help differential fuzzers avoid detecting known/intentional platform-
  // specific differences.
  if (v8_flags.correctness_fuzzer_suppressions) {
    if (detected_features.has_relaxed_simd()) {
      PrintF("Warning: This run cannot be compared across architectures.\n");
    }
  }
}

namespace {

// Returns the start offset of a given import, for use in error messages.
// The module_name payload is preceded by an i32v giving its length. That i32v
// is preceded by another i32v, which is either a type index (specifying the
// type of the previous import) or the imports count (in case of the first
// import). So we scan backwards as long as we find non-last LEB bytes there.
uint32_t ImportStartOffset(base::Vector<const uint8_t> wire_bytes,
                           uint32_t module_name_start) {
  DCHECK_LT(0, module_name_start);
  uint32_t offset = module_name_start - 1;  // Last byte of the string length.
  DCHECK_EQ(wire_bytes[offset] & 0x80, 0);
  while (offset > 0 && (wire_bytes[offset - 1] & 0x80) != 0) {
    offset--;
  }
  return offset;
}

}  // namespace

// Validates the signatures of recognized compile-time imports, and stores
// them on the {module}'s {well_known_imports} list.
WasmError ValidateAndSetBuiltinImports(const WasmModule* module,
                                       base::Vector<const uint8_t> wire_bytes,
                                       const CompileTimeImports& imports,
                                       WasmDetectedFeatures* detected) {
  DCHECK_EQ(module->origin, kWasmOrigin);
  if (imports.empty()) return {};

  std::vector<WellKnownImport> statuses;
  statuses.reserve(module->num_imported_functions);
  for (size_t i = 0; i < module->import_table.size(); i++) {
    const WasmImport& import = module->import_table[i];

    // When magic string imports are requested, check that imports with the
    // string constant module name are globals of the right type.
    if (imports.has_string_constants(wire_bytes.SubVector(
            import.module_name.offset(), import.module_name.end_offset()))) {
      if (import.kind != kExternalGlobal ||
          !module->globals[import.index].type.is_reference_to(
              GenericKind::kExtern) ||
          module->globals[import.index].mutability != false) {
        TruncatedUserString<> name(
            wire_bytes.data() + import.field_name.offset(),
            import.field_name.length());
        return WasmError(
            ImportStartOffset(wire_bytes, import.module_name.offset()),
            "String constant import #%zu \"%.*s\" must be an immutable global "
            "subtyping externref",
            i, name.length(), name.start());
      }
    }

    // Check compile-time imported functions.
    if (import.kind != kExternalFunction &&
        import.kind != kExternalExactFunction) {
      continue;
    }
    base::Vector<const uint8_t> module_name = wire_bytes.SubVector(
        import.module_name.offset(), import.module_name.end_offset());
    constexpr size_t kMinInterestingLength = 10;
    if (module_name.size() < kMinInterestingLength ||
        module_name.SubVector(0, 5) != base::StaticOneByteVector("wasm:")) {
      statuses.push_back(WellKnownImport::kUninstantiated);
      continue;
    }
    base::Vector<const uint8_t> collection = module_name.SubVectorFrom(5);
    WellKnownImport status = WellKnownImport::kUninstantiated;
    const WasmFunction& func = module->functions[import.index];
    const CanonicalTypeIndex sig = module->canonical_sig_id(func.sig_index);
    WireBytesRef field_name = import.field_name;
    base::Vector<const uint8_t> name =
        wire_bytes.SubVector(field_name.offset(), field_name.end_offset());

#define CHECK_SIG(import_name, expected_sig, kEnumName, feature)              \
  if (name == base::StaticOneByteVector(#import_name)) {                      \
    if (V8_UNLIKELY(sig !=                                                    \
                    TypeCanonicalizer::kPredefinedSigIndex_##expected_sig)) { \
      uint32_t error_offset =                                                 \
          ImportStartOffset(wire_bytes, import.module_name.offset());         \
      return WasmError(                                                       \
          error_offset,                                                       \
          "Imported builtin function \"wasm:%s\" \"%s\" has incorrect "       \
          "signature",                                                        \
          std::string(collection.begin(), collection.end()).c_str(),          \
          std::string(name.begin(), name.end()).c_str());                     \
    }                                                                         \
    status = WellKnownImport::kEnumName;                                      \
    detected->add_##feature();                                                \
    break;                                                                    \
  }

    // Not a loop; we just want to use `break` instead of very long else-if
    // cascades.
    do {
      if (collection == base::StaticOneByteVector("js-string") &&
          imports.contains(CompileTimeImport::kJsString)) {
        CHECK_SIG(cast, e_r, kStringCast, imported_strings)
        CHECK_SIG(test, i_r, kStringTest, imported_strings)
        CHECK_SIG(fromCharCode, e_i, kStringFromCharCode, imported_strings)
        CHECK_SIG(fromCodePoint, e_i, kStringFromCodePoint, imported_strings)
        CHECK_SIG(charCodeAt, i_ri, kStringCharCodeAt, imported_strings)
        CHECK_SIG(codePointAt, i_ri, kStringCodePointAt, imported_strings)
        CHECK_SIG(length, i_r, kStringLength, imported_strings)
        CHECK_SIG(concat, e_rr, kStringConcat, imported_strings)
        CHECK_SIG(substring, e_rii, kStringSubstring, imported_strings)
        CHECK_SIG(equals, i_rr, kStringEquals, imported_strings)
        CHECK_SIG(compare, i_rr, kStringCompare, imported_strings)
        CHECK_SIG(fromCharCodeArray, e_a16ii, kStringFromWtf16Array,
                  imported_strings)
        CHECK_SIG(intoCharCodeArray, i_ra16i, kStringToWtf16Array,
                  imported_strings)

      } else if (collection == base::StaticOneByteVector("text-encoder") &&
                 imports.contains(CompileTimeImport::kTextEncoder)) {
        CHECK_SIG(measureStringAsUTF8, i_r, kStringMeasureUtf8,
                  imported_strings_utf8)
        CHECK_SIG(encodeStringIntoUTF8Array, i_ra8i, kStringIntoUtf8Array,
                  imported_strings_utf8)
        CHECK_SIG(encodeStringToUTF8Array, n8_r, kStringToUtf8Array,
                  imported_strings_utf8)

      } else if (collection == base::StaticOneByteVector("text-decoder") &&
                 imports.contains(CompileTimeImport::kTextDecoder)) {
        CHECK_SIG(decodeStringFromUTF8Array, e_a8ii, kStringFromUtf8Array,
                  imported_strings_utf8)

      } else if (collection == base::StaticOneByteVector("js-prototypes") &&
                 imports.contains(CompileTimeImport::kJsPrototypes)) {
        CHECK_SIG(configureAll, configureAll, kConfigureAllPrototypes,
                  custom_descriptors)
      }
    } while (false);
    statuses.push_back(status);
  }
  // We're operating on a fresh WasmModule instance here, so we don't need to
  // check for incompatibilities with previously seen imports.
  DCHECK_EQ(module->num_imported_functions, statuses.size());
  // The "Initialize" call is currently only safe when the decoder has allocated
  // storage, which it allocates when there is an imports section.
  if (module->num_imported_functions != 0) {
    module->type_feedback.well_known_imports.Initialize(
        base::VectorOf(statuses));
  }
  return {};
}

#undef CHECK_SIG

namespace {

enum CompilationExecutionResult : int8_t { kNoMoreUnits, kYield };

const char* GetCompilationEventName(const WasmCompilationUnit& unit,
                                    const CompilationEnv& env) {
  ExecutionTier tier = unit.tier();
  if (tier == ExecutionTier::kLiftoff) {
    return "wasm.BaselineCompilation";
  }
  if (tier == ExecutionTier::kTurbofan) {
    return "wasm.TopTierCompilation";
  }
  if (unit.func_index() <
      static_cast<int>(env.module->num_imported_functions)) {
    return "wasm.WasmToJSWrapperCompilation";
  }
  return "wasm.OtherCompilation";
}

constexpr uint8_t kMainTaskId = 0;

// Run by the {BackgroundCompileJob} (on any thread).
CompilationExecutionResult ExecuteCompilationUnits(
    std::weak_ptr<NativeModule> native_module, JobDelegate* delegate,
    CompilationTier tier) {
  TRACE_EVENT0("v8.wasm", "wasm.ExecuteCompilationUnits");

  // Compilation must be disabled in jitless mode.
  CHECK(!v8_flags.wasm_jitless);

  // These fields are initialized in a {BackgroundCompileScope} before
  // starting compilation.
  std::optional<CompilationEnv> env;
  std::optional<base::FlushDenormalsScope> disable_denormals;
  std::shared_ptr<WireBytesStorage> wire_bytes;
  std::shared_ptr<const WasmModule> module;
  // Task 0 is any main thread (there might be multiple from multiple isolates),
  // worker threads start at 1 (thus the "+ 1").
  static_assert(kMainTaskId == 0);
  int task_id = delegate ? (int{delegate->GetTaskId()} + 1) : kMainTaskId;
  DCHECK_LE(0, task_id);
  CompilationUnitQueues::Queue* queue;
  std::optional<WasmCompilationUnit> unit;
  // Collect histogram samples per compilation thread and merge into the
  // NativeModule on publication of the generated code. We can't directly use
  // the field of the NativeModule as that could die while we compile.
  DelayedCounterUpdates counter_updates;

  WasmDetectedFeatures global_detected_features;

  // Preparation (synchronized): Initialize the fields above and get the first
  // compilation unit.
  {
    BackgroundCompileScope compile_scope(native_module);
    if (compile_scope.cancelled()) return kYield;
    env.emplace(CompilationEnv::ForModule(compile_scope.native_module()));
    // We only really need this for optimized compilation, but for simplicity
    // set it just once for everything.
    disable_denormals.emplace(
        compile_scope.native_module()->compile_imports().contains(
            CompileTimeImport::kDisableDenormalFloats));
    wire_bytes = compile_scope.compilation_state()->GetWireBytesStorage();
    module = compile_scope.native_module()->shared_module();
    queue = compile_scope.compilation_state()->GetQueueForCompileTask(task_id);
    unit =
        compile_scope.compilation_state()->GetNextCompilationUnit(queue, tier);
    if (!unit) return kNoMoreUnits;
  }
  TRACE_COMPILE("ExecuteCompilationUnits (task id %d)\n", task_id);

  std::vector<WasmCompilationResult> results_to_publish;
  while (true) {
    ExecutionTier current_tier = unit->tier();
    const char* event_name = GetCompilationEventName(unit.value(), env.value());
    TRACE_EVENT0("v8.wasm", event_name);
    while (unit->tier() == current_tier) {
      // Track detected features on a per-function basis before collecting them
      // into {global_detected_features}.
      WasmDetectedFeatures per_function_detected_features;
      // (asynchronous): Execute the compilation.
      WasmCompilationResult result = unit->ExecuteCompilation(
          &env.value(), wire_bytes.get(), &counter_updates,
          &per_function_detected_features);
      global_detected_features.Add(per_function_detected_features);
      bool compilation_succeeded = result.succeeded();
      ExecutionTier result_tier = result.result_tier;
      // We don't eagerly compile import wrappers any more.
      DCHECK_GE(unit->func_index(), env->module->num_imported_functions);
      results_to_publish.emplace_back(std::move(result));

      bool yield = delegate && delegate->ShouldYield();

      // (synchronized): Publish the compilation result and get the next unit.
      BackgroundCompileScope compile_scope(native_module);
      if (compile_scope.cancelled()) return kYield;

      if (!compilation_succeeded) {
        compile_scope.compilation_state()->SetError();
        return kNoMoreUnits;
      }

      if (!unit->for_debugging() && result_tier != current_tier) {
        compile_scope.native_module()->AddLiftoffBailout();
      }

      // Yield or get next unit.
      if (yield ||
          !(unit = compile_scope.compilation_state()->GetNextCompilationUnit(
                queue, tier))) {
        std::vector<UnpublishedWasmCode> unpublished_code =
            compile_scope.native_module()->AddCompiledCode(
                base::VectorOf(results_to_publish));
        compile_scope.native_module()->counter_updates()->AddAll(
            std::move(counter_updates));
        DCHECK_EQ(0, counter_updates.NumOutstandingUpdates());
        results_to_publish.clear();
        compile_scope.compilation_state()->SchedulePublishCompilationResults(
            std::move(unpublished_code), tier);
        compile_scope.compilation_state()->OnCompilationStopped(
            global_detected_features);
        return yield ? kYield : kNoMoreUnits;
      }

      // Publish after finishing a certain amount of units, to avoid
      // contention when all threads publish at the end.
      bool batch_full =
          queue->ShouldPublish(static_cast<int>(results_to_publish.size()));
      // Also publish each time the compilation tier changes from Liftoff to
      // TurboFan, such that we immediately publish the baseline compilation
      // results to start execution, and do not wait for a batch to fill up.
      bool liftoff_finished = unit->tier() != current_tier &&
                              unit->tier() == ExecutionTier::kTurbofan;
      if (batch_full || liftoff_finished) {
        std::vector<UnpublishedWasmCode> unpublished_code =
            compile_scope.native_module()->AddCompiledCode(
                base::VectorOf(results_to_publish));
        results_to_publish.clear();
        compile_scope.compilation_state()->SchedulePublishCompilationResults(
            std::move(unpublished_code), tier);
      }
    }
  }
  UNREACHABLE();
}

std::unique_ptr<CompilationUnitBuilder> InitializeCompilation(
    NativeModule* native_module, ProfileInformation* pgo_info) {
  CompilationStateImpl* compilation_state =
      Impl(native_module->compilation_state());
  auto builder = std::make_unique<CompilationUnitBuilder>(native_module);
  compilation_state->InitializeCompilationProgress(pgo_info);
  return builder;
}

WasmError ValidateFunctions(const WasmModule* module,
                            base::Vector<const uint8_t> wire_bytes,
                            WasmEnabledFeatures enabled_features,
                            OnlyLazyFunctions only_lazy_functions,
                            WasmDetectedFeatures* detected_features) {
  DCHECK_EQ(module->origin, kWasmOrigin);
  if (only_lazy_functions && !IsLazyModule(module)) {
    return {};
  }

  std::function<bool(int)> filter;  // Initially empty for "all functions".
  if (only_lazy_functions && enabled_features.has_compilation_hints()) {
    DCHECK(IsLazyModule(module));
    filter = [module](int func_index) {
      return !module->compilation_priorities.contains(func_index);
    };
  }
  // Call {ValidateFunctions} in the module decoder.
  return ValidateFunctions(module, enabled_features, wire_bytes, filter,
                           detected_features);
}

WasmError ValidateFunctions(const NativeModule& native_module,
                            OnlyLazyFunctions only_lazy_functions) {
  WasmDetectedFeatures detected_features;
  WasmError result =
      ValidateFunctions(native_module.module(), native_module.wire_bytes(),
                        native_module.enabled_features(), only_lazy_functions,
                        &detected_features);
  if (!result.has_error()) {
    // This function is called before the NativeModule is finished; all detected
    // features will be published afterwards anyway, so ignore the return value
    // here.
    USE(native_module.compilation_state()->UpdateDetectedFeatures(
        detected_features));
  }
  return result;
}

void CompileNativeModule(ErrorThrower* thrower,
                         std::shared_ptr<NativeModule> native_module,
                         ProfileInformation* pgo_info) {
  CHECK(!v8_flags.jitless || v8_flags.wasm_jitless);
  const WasmModule* module = native_module->module();

  auto* compilation_state = Impl(native_module->compilation_state());

  // Initialize the compilation units and kick off background compile tasks.
  std::unique_ptr<CompilationUnitBuilder> builder =
      InitializeCompilation(native_module.get(), pgo_info);
  compilation_state->InitializeCompilationUnits(std::move(builder));

  // Validate wasm modules for lazy compilation if requested. Never validate
  // asm.js modules as these are valid by construction (additionally a CHECK
  // will catch this during lazy compilation).
  if (!v8_flags.wasm_lazy_validation && module->origin == kWasmOrigin) {
    DCHECK(!thrower->error());
    if (WasmError validation_error =
            ValidateFunctions(*native_module, kOnlyLazyFunctions)) {
      thrower->CompileFailed(std::move(validation_error));
      return;
    }
  }

  if (!compilation_state->failed()) {
    compilation_state->WaitForBaselineCompileJob();
  }

  if (compilation_state->failed()) {
    DCHECK_IMPLIES(IsLazyModule(module), !v8_flags.wasm_lazy_validation);
    WasmError validation_error =
        ValidateFunctions(*native_module, kAllFunctions);
    CHECK(validation_error.has_error());
    thrower->CompileFailed(std::move(validation_error));
  }
}

class BackgroundCompileJob final : public JobTask {
 public:
  explicit BackgroundCompileJob(std::weak_ptr<NativeModule> native_module,
                                CompilationTier tier)
      : native_module_(std::move(native_module)),
        engine_barrier_(GetWasmEngine()->GetBarrierForBackgroundCompile()),
        tier_(tier) {}

  void Run(JobDelegate* delegate) override {
    auto engine_scope = engine_barrier_->TryLock();
    if (!engine_scope) return;
    ExecuteCompilationUnits(native_module_, delegate, tier_);
  }

  size_t GetMaxConcurrency(size_t worker_count) const override {
    BackgroundCompileScope compile_scope(native_module_);
    if (compile_scope.cancelled()) return 0;
    size_t flag_limit = static_cast<size_t>(
        std::max(1, v8_flags.wasm_num_compilation_tasks.value()));
    // NumOutstandingCompilations() does not reflect the units that running
    // workers are processing, thus add the current worker count to that number.
    return std::min(flag_limit,
                    worker_count + compile_scope.compilation_state()
                                       ->NumOutstandingCompilations(tier_));
  }

 private:
  std::weak_ptr<NativeModule> native_module_;
  std::shared_ptr<OperationsBarrier> engine_barrier_;
  const CompilationTier tier_;
};

std::shared_ptr<NativeModule> GetOrCompileNewNativeModule(
    Isolate* isolate, WasmEnabledFeatures enabled_features,
    WasmDetectedFeatures detected_features, CompileTimeImports compile_imports,
    ErrorThrower* thrower, std::shared_ptr<const WasmModule> module,
    base::OwnedVector<const uint8_t> wire_bytes, int compilation_id,
    v8::metrics::Recorder::ContextId context_id, ProfileInformation* pgo_info) {
  base::TimeTicks start_time;
  if (base::TimeTicks::IsHighResolution()) start_time = base::TimeTicks::Now();

  std::shared_ptr<NativeModule> native_module =
      GetWasmEngine()->MaybeGetNativeModule(
          module->origin, wire_bytes.as_vector(), compile_imports);
  if (native_module) {
    GetWasmEngine()->UseNativeModuleInIsolate(native_module.get(), isolate);

    v8::metrics::WasmModuleCompiled compile_event{
        .async = false,
        .streamed = false,
        .cached = true,
        .deserialized = false,
        .lazy = v8_flags.wasm_lazy_compilation,
        .success = true,
        .code_size_in_bytes = native_module->generated_code_size(),
        .liftoff_bailout_count = native_module->liftoff_bailout_count()};
    if (!start_time.IsNull()) {
      base::TimeDelta duration = base::TimeTicks::Now() - start_time;
      compile_event.wall_clock_duration_in_us = duration.InMicroseconds();
    }
    isolate->metrics_recorder()->DelayMainThreadEvent(compile_event,
                                                      context_id);
    return native_module;
  }

  // Otherwise compile a new NativeModule.
  size_t code_size_estimate =
      wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module.get());
  native_module = GetWasmEngine()->NewNativeModule(
      isolate, enabled_features, detected_features, std::move(compile_imports),
      module, code_size_estimate);
  native_module->SetWireBytes(std::move(wire_bytes));
  native_module->compilation_state()->set_compilation_id(compilation_id);

  if (!v8_flags.wasm_jitless) {
    // Compile / validate the new module.
    CompileNativeModule(thrower, native_module, pgo_info);
  }

  bool failed = thrower->error();
  // Finally, put the new module in the cache; this can return the passed
  // NativeModule pointer, or another one (for a previously cached module).
  native_module = GetWasmEngine()->UpdateNativeModuleCache(
      failed, std::move(native_module), isolate);

  v8::metrics::WasmModuleCompiled compile_event{
      .async = false,
      .streamed = false,
      .cached = false,
      .deserialized = false,
      .lazy = v8_flags.wasm_lazy_compilation,
      .success = !failed,
      .code_size_in_bytes = native_module->generated_code_size(),
      .liftoff_bailout_count = native_module->liftoff_bailout_count()};

  if (!start_time.IsNull()) {
    base::TimeDelta duration = base::TimeTicks::Now() - start_time;
    SELECT_WASM_COUNTER(isolate->counters(), module->origin, wasm_compile,
                        module_time)
        ->AddTimedSample(duration);
    compile_event.wall_clock_duration_in_us = duration.InMicroseconds();
  }

  isolate->metrics_recorder()->DelayMainThreadEvent(compile_event, context_id);

  if (failed) native_module = {};
  return native_module;
}

}  // namespace

std::shared_ptr<NativeModule> CompileToNativeModule(
    Isolate* isolate, WasmEnabledFeatures enabled_features,
    WasmDetectedFeatures detected_features, CompileTimeImports compile_imports,
    ErrorThrower* thrower, std::shared_ptr<const WasmModule> module,
    base::OwnedVector<const uint8_t> wire_bytes, int compilation_id,
    v8::metrics::Recorder::ContextId context_id, ProfileInformation* pgo_info) {
  std::shared_ptr<NativeModule> native_module = GetOrCompileNewNativeModule(
      isolate, enabled_features, detected_features, std::move(compile_imports),
      thrower, module, std::move(wire_bytes), compilation_id, context_id,
      pgo_info);
  if (!native_module) return {};

  // Ensure that the code objects are logged before returning.
  GetWasmEngine()->LogOutstandingCodesForIsolate(isolate);

  // Now publish all detected features of this module in the current isolate.
  PublishDetectedFeatures(
      native_module->compilation_state()->detected_features(), isolate, true);

  return native_module;
}

// A closure to run a compilation step (either as foreground or background
// task) and schedule the next step(s), if any.
class AsyncCompileJob::CompileStep {
 public:
  virtual ~CompileStep() = default;

  void Run(AsyncCompileJob* job, bool on_foreground) {
    if (on_foreground) {
      HandleScope scope(job->isolate_specific_info_.isolate_);
      SaveAndSwitchContext saved_context(
          job->isolate_specific_info_.isolate_,
          *job->isolate_specific_info_.native_context_);
      RunInForeground(job);
    } else {
      RunInBackground(job);
    }
  }

  virtual void RunInForeground(AsyncCompileJob*) { UNREACHABLE(); }
  virtual void RunInBackground(AsyncCompileJob*) { UNREACHABLE(); }
};

class AsyncCompileJob::CompileTask : public CancelableTask {
 public:
  CompileTask(AsyncCompileJob* job, bool on_foreground)
      // We only manage the background tasks with the {CancelableTaskManager} of
      // the {AsyncCompileJob}. Foreground tasks are managed by the system's
      // {CancelableTaskManager}. Background tasks cannot spawn tasks managed by
      // their own task manager.
      : CancelableTask(on_foreground ? job->isolate_specific_info_.isolate_
                                           ->cancelable_task_manager()
                                     : &job->background_task_manager_),
        job_(job),
        on_foreground_(on_foreground) {}

  ~CompileTask() override {
    if (job_ != nullptr && on_foreground_) ResetPendingForegroundTask();
  }

  void RunInternal() final {
    if (!job_) return;
    if (on_foreground_) ResetPendingForegroundTask();
    job_->step_->Run(job_, on_foreground_);
    // After execution, reset {job_} such that we don't try to reset the pending
    // foreground task when the task is deleted.
    job_ = nullptr;
  }

  void Cancel() {
    DCHECK_NOT_NULL(job_);
    job_ = nullptr;
  }

 private:
  // {job_} will be cleared to cancel a pending task.
  AsyncCompileJob* job_;
  bool on_foreground_;

  void ResetPendingForegroundTask() const {
    DCHECK_EQ(this, job_->pending_foreground_task_);
    job_->pending_foreground_task_ = nullptr;
  }
};

AsyncCompileJob::AsyncCompileJob(
    WasmEnabledFeatures enabled_features, CompileTimeImports compile_imports,
    base::OwnedVector<const uint8_t> bytes, const char* api_method_name,
    std::shared_ptr<CompilationResultResolver> resolver, int compilation_id)
    : api_method_name_(api_method_name),
      enabled_features_(enabled_features),
      compile_imports_(std::move(compile_imports)),
      start_time_(base::TimeTicks::IsHighResolution() ? base::TimeTicks::Now()
                                                      : base::TimeTicks{}),
      bytes_copy_(std::move(bytes)),
      wire_bytes_(bytes_copy_.as_vector()),
      resolver_(std::move(resolver)),
      compilation_id_(compilation_id) {
  TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
               "wasm.AsyncCompileJob");
  CHECK(v8_flags.wasm_async_compilation);
  CHECK(!v8_flags.jitless || v8_flags.wasm_jitless);
}

void AsyncCompileJob::InitializeIsolateSpecificInfo(Isolate* isolate) {
  DCHECK_EQ(isolate->thread_id(), ThreadId::Current());
  DCHECK(!has_isolate_specific_info_.load(std::memory_order_acquire));
  v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate);
  v8::Platform* platform = V8::GetCurrentPlatform();
  DirectHandle<NativeContext> native_context = isolate->native_context();
  isolate_specific_info_ = IsolateSpecificInfo{
      .isolate_ = isolate,
      .native_context_ = isolate->global_handles()->Create(*native_context),
      .incumbent_context_ =
          isolate->global_handles()->Create(*isolate->GetIncumbentContext()),
      .context_id_ = isolate->GetOrRegisterRecorderContextId(native_context),
      .foreground_task_runner_ = platform->GetForegroundTaskRunner(v8_isolate)};
  CHECK(!has_isolate_specific_info_.exchange(true, std::memory_order_release));
}

void AsyncCompileJob::StartAsyncDecoding() { DoAsync<DecodeModule>(); }

void AsyncCompileJob::Abort() {
  // Removing this job will trigger the destructor, which will cancel all
  // compilation.
  GetWasmEngine()->RemoveCompileJob(this);
}

// {ValidateFunctionsStreamingJobData} holds information that is shared between
// the {AsyncStreamingProcessor} and the {ValidateFunctionsStreamingJob}. It
// lives in the {AsyncStreamingProcessor} and is updated from both classes.
struct ValidateFunctionsStreamingJobData {
  struct Unit {
    // {func_index == -1} represents an "invalid" unit.
    int func_index = -1;
    base::Vector<const uint8_t> code;

    // Check whether the unit is valid.
    operator bool() const {
      DCHECK_LE(-1, func_index);
      return func_index >= 0;
    }
  };

  void Initialize(int num_declared_functions) {
    DCHECK_NULL(units);
    units = base::OwnedVector<Unit>::NewForOverwrite(num_declared_functions);
    // Initially {next == end}.
    next_available_unit.store(units.begin(), std::memory_order_relaxed);
    end_of_available_units.store(units.begin(), std::memory_order_relaxed);
  }

  void AddUnit(int declared_func_index, base::Vector<const uint8_t> code,
               JobHandle* job_handle) {
    DCHECK_NOT_NULL(units);
    // Write new unit to {*end}, then increment {end}. There is only one thread
    // adding new units, so no further synchronization needed.
    Unit* ptr = end_of_available_units.load(std::memory_order_relaxed);
    // Check invariant: {next <= end}.
    DCHECK_LE(next_available_unit.load(std::memory_order_relaxed), ptr);
    *ptr++ = {declared_func_index, code};
    // Use release semantics, so whoever loads this pointer (using acquire
    // semantics) sees all our previous stores.
    end_of_available_units.store(ptr, std::memory_order_release);
    size_t total_units_added = ptr - units.begin();
    // Periodically notify concurrency increase. This has overhead, so avoid
    // calling it too often. As long as threads are still running they will
    // continue processing new units anyway, and if background threads validate
    // faster than we can add units, then only notifying after increasingly long
    // delays is the right thing to do to avoid too many small validation tasks.
    // We notify on each power of two after 16 units, and every 16k units (just
    // to have *some* upper limit and avoiding to pile up too many units).
    // Additionally, notify after receiving the last unit of the module.
    if ((total_units_added >= 16 &&
         base::bits::IsPowerOfTwo(total_units_added)) ||
        (total_units_added % (16 * 1024)) == 0 || ptr == units.end()) {
      job_handle->NotifyConcurrencyIncrease();
    }
  }

  size_t NumOutstandingUnits() const {
    Unit* next = next_available_unit.load(std::memory_order_relaxed);
    Unit* end = end_of_available_units.load(std::memory_order_relaxed);
    DCHECK_LE(next, end);
    return end - next;
  }

  // Retrieve one unit to validate; returns an "invalid" unit if nothing is in
  // the queue.
  Unit GetUnit() {
    // Use an acquire load to synchronize with the store in {AddUnit}. All units
    // before this {end} are fully initialized and ready to execute.
    Unit* end = end_of_available_units.load(std::memory_order_acquire);
    Unit* next = next_available_unit.load(std::memory_order_relaxed);
    while (next < end) {
      if (next_available_unit.compare_exchange_weak(
              next, next + 1, std::memory_order_relaxed)) {
        return *next;
      }
      // Otherwise retry with updated {next} pointer.
    }
    return {};
  }

  void UpdateDetectedFeatures(WasmDetectedFeatures new_detected_features) {
    WasmDetectedFeatures old_features =
        detected_features.load(std::memory_order_relaxed);
    while (!detected_features.compare_exchange_weak(
        old_features, old_features | new_detected_features,
        std::memory_order_relaxed)) {
      // Retry with updated {old_features}.
    }
  }

  base::OwnedVector<Unit> units;
  std::atomic<Unit*> next_available_unit;
  std::atomic<Unit*> end_of_available_units;
  std::atomic<bool> found_error{false};
  std::atomic<WasmDetectedFeatures> detected_features;
};

class ValidateFunctionsStreamingJob final : public JobTask {
 public:
  ValidateFunctionsStreamingJob(const WasmModule* module,
                                WasmEnabledFeatures enabled_features,
                                ValidateFunctionsStreamingJobData* data)
      : module_(module), enabled_features_(enabled_features), data_(data) {}

  void Run(JobDelegate* delegate) override {
    TRACE_EVENT0("v8.wasm", "wasm.ValidateFunctionsStreaming");
    using Unit = ValidateFunctionsStreamingJobData::Unit;
    Zone validation_zone{GetWasmEngine()->allocator(),
                         "Wasm ValidateFunctionsStreamingJob"};
    WasmDetectedFeatures detected_features;
    while (Unit unit = data_->GetUnit()) {
      validation_zone.Reset();
      DecodeResult result = ValidateSingleFunction(
          &validation_zone, module_, unit.func_index, unit.code,
          enabled_features_, &detected_features);

      if (result.failed()) {
        data_->found_error.store(true, std::memory_order_relaxed);
        break;
      }
      // After validating one function, check if we should yield.
      if (delegate->ShouldYield()) break;
    }

    data_->UpdateDetectedFeatures(detected_features);
  }

  size_t GetMaxConcurrency(size_t worker_count) const override {
    return worker_count + data_->NumOutstandingUnits();
  }

 private:
  const WasmModule* const module_;
  const WasmEnabledFeatures enabled_features_;
  ValidateFunctionsStreamingJobData* data_;
};

class AsyncStreamingProcessor final : public StreamingProcessor {
 public:
  explicit AsyncStreamingProcessor(AsyncCompileJob* job);

  void InitializeIsolateSpecificInfo(Isolate* isolate) override {
    job_->InitializeIsolateSpecificInfo(isolate);
  }

  bool ProcessModuleHeader(base::Vector<const uint8_t> bytes) override;

  bool ProcessSection(SectionCode section_code,
                      base::Vector<const uint8_t> bytes,
                      uint32_t offset) override;

  bool ProcessCodeSectionHeader(int num_functions,
                                uint32_t functions_mismatch_error_offset,
                                std::shared_ptr<WireBytesStorage>,
                                size_t code_section_start,
                                size_t code_section_length) override;

  bool ProcessFunctionBody(base::Vector<const uint8_t> bytes,
                           uint32_t offset) override;

  void OnFinishedChunk() override;

  void OnFinishedStream(base::OwnedVector<const uint8_t> bytes,
                        bool after_error) override;

  void OnAbort() override;

  bool Deserialize(base::Vector<const uint8_t> module_bytes,
                   base::OwnedVector<const uint8_t>& wire_bytes) override;

 private:
  void CommitCompilationUnits();

  ModuleDecoder decoder_;
  AsyncCompileJob* job_;
  std::unique_ptr<CompilationUnitBuilder> compilation_unit_builder_;
  int num_functions_ = 0;
  bool prefix_cache_hit_ = false;
  bool before_code_section_ = true;
  ValidateFunctionsStreamingJobData validate_functions_job_data_;
  std::unique_ptr<JobHandle> validate_functions_job_handle_;

  // {prefix_hasher_} computes a running hash of the wire bytes up to code
  // section size, but excludes the code section itself. Used by the
  // {NativeModuleCache} to detect potential duplicate modules.
  base::Hasher prefix_hasher_;
};

std::shared_ptr<StreamingDecoder> AsyncCompileJob::CreateStreamingDecoder() {
  DCHECK_NULL(stream_);
  stream_ = StreamingDecoder::CreateAsyncStreamingDecoder(
      std::make_unique<AsyncStreamingProcessor>(this));
  return stream_;
}

AsyncCompileJob::~AsyncCompileJob() {
  // Note: This destructor always runs on the foreground thread of the isolate.
  background_task_manager_.CancelAndWait();
  // If initial compilation did not finish yet we can abort it.
  if (new_native_module_) {
    Impl(new_native_module_->compilation_state())
        ->CancelCompilation(CompilationStateImpl::kCancelInitialCompilation);
  }
  // Tell the streaming decoder that the AsyncCompileJob is not available
  // anymore.
  if (stream_) stream_->NotifyCompilationDiscarded();
  CancelPendingForegroundTask();
  if (isolate_specific_info_.isolate_) {
    DCHECK_EQ(isolate_specific_info_.isolate_->thread_id(),
              ThreadId::Current());
    isolate_specific_info_.isolate_->global_handles()->Destroy(
        isolate_specific_info_.native_context_.location());
    isolate_specific_info_.isolate_->global_handles()->Destroy(
        isolate_specific_info_.incumbent_context_.location());
  }
}

void AsyncCompileJob::CreateNativeModule(
    std::shared_ptr<const WasmModule> module, size_t code_size_estimate) {
  DCHECK_NULL(new_native_module_);
  new_native_module_ = GetWasmEngine()->NewUnownedNativeModule(
      enabled_features_, detected_features_, std::move(compile_imports_),
      std::move(module), code_size_estimate);
  new_native_module_->SetWireBytes(std::move(bytes_copy_));
  new_native_module_->compilation_state()->set_compilation_id(compilation_id_);
}

std::tuple<std::shared_ptr<NativeModule>, bool>
AsyncCompileJob::GetOrCreateNativeModule(
    std::shared_ptr<const WasmModule> module, size_t code_size_estimate) {
  DCHECK_NULL(new_native_module_);
  std::shared_ptr<NativeModule> cached_native_module =
      GetWasmEngine()->MaybeGetNativeModule(
          module->origin, wire_bytes_.module_bytes(), compile_imports_);
  if (cached_native_module) return {cached_native_module, true};
  CreateNativeModule(std::move(module), code_size_estimate);
  return {new_native_module_, false};
}

// This function assumes that it is executed in a HandleScope, and that a
// context is set on the isolate.
void AsyncCompileJob::FinishCompile(
    std::shared_ptr<NativeModule> native_module,
    DirectHandle<WasmModuleObject> deserialized_module_object,
    bool cache_hit) && {
  TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
               "wasm.FinishAsyncCompile");
  Isolate* isolate = isolate_specific_info_.isolate_;
  DCHECK_EQ(isolate->thread_id(), ThreadId::Current());
  GetWasmEngine()->UseNativeModuleInIsolate(native_module.get(), isolate);
  if (stream_) {
    stream_->NotifyNativeModuleCreated(native_module);
  }
  const WasmModule* module = native_module->module();
  auto compilation_state = Impl(native_module->compilation_state());

  // Update the compilation state with feature detected during module decoding
  // and (potentially) validation. We will publish all features below, in the
  // current isolate, so ignore the return value here.
  USE(compilation_state->UpdateDetectedFeatures(detected_features_));

  // If experimental PGO via files is enabled, load profile information now that
  // we have all wire bytes and know that the module is valid.
  if (V8_UNLIKELY(v8_flags.experimental_wasm_pgo_from_file)) {
    std::unique_ptr<ProfileInformation> pgo_info =
        LoadProfileFromFile(module, native_module->wire_bytes());
    if (pgo_info) {
      compilation_state->ApplyPgoInfoLate(pgo_info.get());
    }
  }

  DirectHandle<WasmModuleObject> module_object = deserialized_module_object;
  bool is_after_deserialization = !deserialized_module_object.is_null();
  if (!is_after_deserialization) {
    // Prepare runtime objects.
    auto source_url =
        stream_ ? base::VectorOf(stream_->url()) : base::Vector<const char>();
    auto script =
        GetWasmEngine()->GetOrCreateScript(isolate, native_module, source_url);
    module_object = WasmModuleObject::New(isolate, native_module, script);
  }

  // We should only get here if compilation succeeded.
  DCHECK(!compilation_state->failed());
  v8::metrics::WasmModuleCompiled compile_event{
      .async = true,
      .streamed = stream_ != nullptr,
      .cached = cache_hit,
      .deserialized = is_after_deserialization,
      .lazy = v8_flags.wasm_lazy_compilation,
      .success = true,
      .code_size_in_bytes = native_module->generated_code_size(),
      .liftoff_bailout_count = native_module->liftoff_bailout_count()};

  if (!start_time_.IsNull()) {
    Counters* counters = isolate->counters();
    // We will only have a compilation time if we did not get the module from
    // cache and if there was a code section.
    if (!compilation_finished_time_.IsNull()) {
      base::TimeDelta compile_time = compilation_finished_time_ - start_time_;
      auto* histogram =
          stream_ ? counters->wasm_streaming_compile_wasm_module_time()
                  : counters->wasm_async_compile_wasm_module_time();
      histogram->AddTimedSample(compile_time);
    }

    base::TimeDelta finish_time = base::TimeTicks::Now() - start_time_;
    // For streaming compilation, we distinguish between "compile time" and
    // "finish time" which in particular includes waiting for the download to
    // finish (but also e.g. lookup in the native module cache).
    if (stream_) {
      counters->wasm_streaming_finish_wasm_module_time()->AddTimedSample(
          finish_time);
    }

    // The UKM event reports the full time until the module is ready.
    compile_event.wall_clock_duration_in_us = finish_time.InMicroseconds();
  }
  delayed_counters_.Publish(isolate);

  DCHECK(!isolate->context().is_null());
  auto context_id =
      isolate->GetOrRegisterRecorderContextId(isolate->native_context());
  isolate->metrics_recorder()->AddMainThreadEvent(compile_event, context_id);
  // If we have a decoding metrics event (from non-streaming compilation),
  // publish it now.
  if (decoding_metrics_event_.has_value()) {
    isolate->metrics_recorder()->AddMainThreadEvent(
        decoding_metrics_event_.value(), context_id);
  }

  // Finish the wasm script now and make it public to the debugger.
  DirectHandle<Script> script(module_object->script(), isolate);
  auto sourcemap_symbol =
      module->debug_symbols[WasmDebugSymbols::Type::SourceMap];
  if (script->type() == Script::Type::kWasm &&
      sourcemap_symbol.type != WasmDebugSymbols::Type::None &&
      !sourcemap_symbol.external_url.is_empty()) {
    ModuleWireBytes wire_bytes(native_module->wire_bytes());
    MaybeDirectHandle<String> src_map_str =
        isolate->factory()->NewStringFromUtf8(
            wire_bytes.GetNameOrNull(sourcemap_symbol.external_url),
            AllocationType::kOld);
    script->set_source_mapping_url(*src_map_str.ToHandleChecked());
  }
  {
    TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
                 "wasm.Debug.OnAfterCompile");
    isolate->debug()->OnAfterCompile(script);
  }

  // Publish the detected features in this isolate, once initial compilation
  // is done. Validate should have detected all features, unless lazy validation
  // is enabled.
  PublishDetectedFeatures(compilation_state->detected_features(), isolate,
                          true);
  // Also publish any delayed counter updates in the isolate.
  native_module->counter_updates()->Publish(isolate);

  // We might need debug code for the module, if the debugger was enabled while
  // streaming compilation was running. Since handling this while compiling via
  // streaming is tricky, we just remove all code which may have been generated,
  // and compile debug code lazily.
  if (native_module->IsInDebugState()) {
    WasmCodeRefScope ref_scope;
    native_module->RemoveCompiledCode(
        NativeModule::RemoveFilter::kRemoveNonDebugCode);
  }

  // Finally, log all generated code (it does not matter if this happens
  // repeatedly in case the script is shared).
  native_module->LogWasmCodes(isolate, *script);

  {
    TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
                 "wasm.OnCompilationSucceeded");
    // We have to make sure that an "incumbent context" is available in case the
    // module's start function calls out to Blink.
    Local<v8::Context> backup_incumbent_context =
        Utils::ToLocal(isolate_specific_info_.incumbent_context_);
    v8::Context::BackupIncumbentScope incumbent(backup_incumbent_context);
    resolver_->OnCompilationSucceeded(module_object);
  }
  GetWasmEngine()->RemoveCompileJob(this);
}

void AsyncCompileJob::Failed() && {
  // {job} keeps the {this} pointer alive.
  std::unique_ptr<AsyncCompileJob> job =
      GetWasmEngine()->RemoveCompileJob(this);

  Isolate* isolate = isolate_specific_info_.isolate_;
  DCHECK_EQ(isolate, Isolate::Current());

  if (new_native_module_) {
    // Publish any delayed counter updates for the "unfinished" native module in
    // the isolate.
    new_native_module_->counter_updates()->Publish(isolate);
  }

  // Revalidate the whole module to produce a deterministic error message.
  constexpr bool kValidate = true;
  WasmDetectedFeatures unused_detected_features;
  ModuleResult result =
      DecodeWasmModule(enabled_features_, wire_bytes_.module_bytes(), kValidate,
                       kWasmOrigin, &unused_detected_features);
  ErrorThrower thrower(isolate, api_method_name_);
  if (result.failed()) {
    thrower.CompileFailed(std::move(result).error());
  } else {
    // The only possible reason why {result} might be okay is if the failure
    // was due to compile-time imports checking.
    CHECK(!job->compile_imports_.empty());
    WasmError error = ValidateAndSetBuiltinImports(
        result.value().get(), wire_bytes_.module_bytes(), job->compile_imports_,
        &unused_detected_features);
    CHECK(error.has_error());
    thrower.CompileError("%s", error.message().c_str());
  }

  v8::metrics::WasmModuleCompiled compile_event{
      .async = true,
      .streamed = stream_ != nullptr,
      .cached = false,
      .deserialized = false,
      .lazy = v8_flags.wasm_lazy_compilation,
      .success = false,
      .code_size_in_bytes = 0,
      .liftoff_bailout_count = 0};
  if (!start_time_.IsNull()) {
    base::TimeDelta duration = base::TimeTicks::Now() - start_time_;
    compile_event.wall_clock_duration_in_us = duration.InMicroseconds();
  }
  auto context_id =
      isolate->GetOrRegisterRecorderContextId(isolate->native_context());
  isolate->metrics_recorder()->AddMainThreadEvent(compile_event, context_id);
  // If we have a decoding metrics event (from non-streaming compilation),
  // publish it now.
  if (decoding_metrics_event_.has_value()) {
    isolate->metrics_recorder()->AddMainThreadEvent(
        decoding_metrics_event_.value(), context_id);
  }

  job->delayed_counters_.Publish(isolate);

  resolver_->OnCompilationFailed(thrower.Reify());
}

class AsyncCompileJob::CompilationStateCallback
    : public CompilationEventCallback {
 public:
  explicit CompilationStateCallback(AsyncCompileJob* job) : job_(job) {}

  void call(CompilationEvent event) override {
    switch (event) {
      case CompilationEvent::kFinishedBaselineCompilation:
        DCHECK(!last_event_.has_value());
        if (base::TimeTicks::IsHighResolution()) {
          job_->compilation_finished_time_ = base::TimeTicks::Now();
        }
        if (job_->DecrementAndCheckFinisherCount()) {
          // Install the native module in the cache, or reuse a conflicting one.
          std::shared_ptr<NativeModule> final_native_module =
              GetWasmEngine()->UpdateNativeModuleCache(
                  false, job_->new_native_module_,
                  job_->isolate_specific_info_.isolate_);
          job_->DoSync<FinishCompilation>(std::move(final_native_module));
        }
        break;
      case CompilationEvent::kFinishedCompilationChunk:
        DCHECK(CompilationEvent::kFinishedBaselineCompilation == last_event_ ||
               CompilationEvent::kFinishedCompilationChunk == last_event_);
        break;
      case CompilationEvent::kFailedCompilation:
        DCHECK(!last_event_.has_value());
        if (job_->DecrementAndCheckFinisherCount()) {
          GetWasmEngine()->UpdateNativeModuleCache(
              true, job_->new_native_module_,
              job_->isolate_specific_info_.isolate_);
          job_->DoSync<Fail>();
        }
        break;
    }
#ifdef DEBUG
    last_event_ = event;
#endif
  }

 private:
  AsyncCompileJob* job_;
#ifdef DEBUG
  // This will be modified by different threads, but they externally
  // synchronize, so no explicit synchronization (currently) needed here.
  std::optional<CompilationEvent> last_event_;
#endif
};

void AsyncCompileJob::StartForegroundTask() {
  DCHECK_NULL(pending_foreground_task_);

  auto new_task = std::make_unique<CompileTask>(this, true);
  pending_foreground_task_ = new_task.get();
  isolate_specific_info_.foreground_task_runner_->PostTask(std::move(new_task));
}

void AsyncCompileJob::CancelPendingForegroundTask() {
  if (!pending_foreground_task_) return;
  pending_foreground_task_->Cancel();
  pending_foreground_task_ = nullptr;
}

void AsyncCompileJob::StartBackgroundTask() {
  auto task = std::make_unique<CompileTask>(this, false);

  // If --wasm-num-compilation-tasks=0 is passed, do only spawn foreground
  // tasks. This is used to make timing deterministic.
  if (v8_flags.wasm_num_compilation_tasks > 0) {
    V8::GetCurrentPlatform()->PostTaskOnWorkerThread(
        TaskPriority::kUserBlocking, std::move(task));
  } else {
    isolate_specific_info_.foreground_task_runner_->PostTask(std::move(task));
  }
}

template <typename Step,
          AsyncCompileJob::UseExistingForegroundTask use_existing_fg_task,
          typename... Args>
void AsyncCompileJob::DoSync(Args&&... args) {
  NextStep<Step>(std::forward<Args>(args)...);
  if (use_existing_fg_task && pending_foreground_task_ != nullptr) return;
  StartForegroundTask();
}

template <typename Step, typename... Args>
void AsyncCompileJob::DoAsync(Args&&... args) {
  NextStep<Step>(std::forward<Args>(args)...);
  StartBackgroundTask();
}

template <typename Step, typename... Args>
void AsyncCompileJob::NextStep(Args&&... args) {
  step_.reset(new Step(std::forward<Args>(args)...));
}

//==========================================================================
// Step 1: (async) Decode the module.
//==========================================================================
class AsyncCompileJob::DecodeModule : public AsyncCompileJob::CompileStep {
 public:
  void RunInBackground(AsyncCompileJob* job) override {
    DisallowHandleAllocation no_handle;
    DisallowHeapAllocation no_allocation;
    // Decode the module bytes.
    TRACE_COMPILE("(1) Decoding module...\n");
    TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
                 "wasm.DecodeModule");
    auto enabled_features = job->enabled_features_;
    ModuleResult result = DecodeWasmModule(
        enabled_features, job->wire_bytes_.module_bytes(), false, kWasmOrigin,
        &job->delayed_counters_, &job->decoding_metrics_event_,
        DecodingMethod::kAsync, &job->detected_features_);

    // Validate lazy functions here if requested.
    if (result.ok() && !v8_flags.wasm_lazy_validation) {
      const WasmModule* module = result.value().get();
      if (WasmError validation_error = ValidateFunctions(
              module, job->wire_bytes_.module_bytes(), job->enabled_features_,
              kOnlyLazyFunctions, &job->detected_features_)) {
        result = ModuleResult{std::move(validation_error)};
      }
    }
    if (result.ok()) {
      const WasmModule* module = result.value().get();
      if (WasmError error = ValidateAndSetBuiltinImports(
              module, job->wire_bytes_.module_bytes(), job->compile_imports_,
              &job->detected_features_)) {
        result = ModuleResult{std::move(error)};
      }
    }
    if (result.failed()) {
      // Decoding failure; reject the promise and clean up.
      job->DoSync<Fail>();
    } else {
      // Decode passed.
      std::shared_ptr<WasmModule> module = std::move(result).value();
      size_t code_size_estimate =
          wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module.get());
      job->DoAsync<PrepareNativeModule>(
          std::move(module), true /* start_compilation */,
          true /* lazy_functions_are_validated */, code_size_estimate);
    }
  }
};

//==========================================================================
// Step 2 (async): Allocates NativeModule and potentially starts background
// compilation.
//==========================================================================
class AsyncCompileJob::PrepareNativeModule : public CompileStep {
 public:
  PrepareNativeModule(std::shared_ptr<const WasmModule> module,
                      bool start_compilation, bool lazy_functions_are_validated,
                      size_t code_size_estimate)
      : module_(std::move(module)),
        start_compilation_(start_compilation),
        lazy_functions_are_validated_(lazy_functions_are_validated),
        code_size_estimate_(code_size_estimate) {}

  void RunInBackground(AsyncCompileJob* job) override {
    TRACE_COMPILE("(2) Prepare and start compile...\n");

    const bool streaming = job->wire_bytes_.length() == 0;

    std::shared_ptr<NativeModule> final_native_module;
    bool cache_hit = false;

    if (streaming) {
      // Streaming compilation already checked for cache hits.
      job->CreateNativeModule(module_, code_size_estimate_);
      final_native_module = job->new_native_module_;
    } else {
      std::tie(final_native_module, cache_hit) =
          job->GetOrCreateNativeModule(std::move(module_), code_size_estimate_);
    }

    if (cache_hit) {
      DCHECK(!streaming);
      // Finish compilation synchronously.
      // Pass the native module here so we know that this was a cache hit and
      // report this properly in UKM events.
      job->DoSync<FinishCompilation>(std::move(final_native_module));
      return;
    }

    if (!streaming && !lazy_functions_are_validated_) {
      // If we are not streaming and did not get a cache hit, we might have hit
      // the path where the streaming decoder got a prefix cache hit, but the
      // module then turned out to be invalid, and we are running it through
      // non-streaming decoding again. In this case, function bodies have not
      // been validated yet (would have happened in the {DecodeModule} phase
      // if we would not come via the non-streaming path). Thus do this now.
      // Note that we only need to validate lazily compiled functions, others
      // will be validated during eager compilation.
      DCHECK(start_compilation_);
      if (!v8_flags.wasm_lazy_validation &&
          ValidateFunctions(*final_native_module, kOnlyLazyFunctions)
              .has_error()) {
        // Fail compilation synchronously.
        job->DoSync<Fail>();
        return;
      }
    }

    CompilationStateImpl* compilation_state =
        Impl(job->new_native_module_->compilation_state());
    compilation_state->AddCallback(
        std::make_unique<CompilationStateCallback>(job));

    if (start_compilation_) {
      // TODO(13209): Use PGO for async compilation, if available.
      constexpr ProfileInformation* kNoProfileInformation = nullptr;
      std::unique_ptr<CompilationUnitBuilder> builder = InitializeCompilation(
          final_native_module.get(), kNoProfileInformation);

      compilation_state->InitializeCompilationUnits(std::move(builder));
      // In single-threaded mode there are no worker tasks that will do the
      // compilation. We call {WaitForCompilationEvent} here so that the current
      // thread participates and finishes the compilation.
      if (v8_flags.wasm_num_compilation_tasks == 0 || v8_flags.wasm_jitless) {
        compilation_state->WaitForBaselineCompileJob();
      }
    }
  }

 private:
  const std::shared_ptr<const WasmModule> module_;
  const bool start_compilation_;
  const bool lazy_functions_are_validated_;
  const size_t code_size_estimate_;
};

//==========================================================================
// Step 3 (sync): Compilation finished.
//==========================================================================
class AsyncCompileJob::FinishCompilation : public CompileStep {
 public:
  explicit FinishCompilation(std::shared_ptr<NativeModule> final_native_module)
      : final_native_module_(std::move(final_native_module)) {}

 private:
  void RunInForeground(AsyncCompileJob* job) override {
    TRACE_COMPILE("(3) Compilation finished\n");
    DCHECK_EQ(job->isolate_specific_info_.isolate_, Isolate::Current());
    // Then finalize and publish the generated module.
    // This invalidates the {AsyncCompileJob}.
    bool cache_hit = final_native_module_ != job->new_native_module_;
    std::move(*job).FinishCompile(std::move(final_native_module_), {},
                                  cache_hit);
  }

  std::shared_ptr<NativeModule> final_native_module_;
};

//==========================================================================
// Step 4 (sync): Decoding or compilation failed.
//==========================================================================
class AsyncCompileJob::Fail : public CompileStep {
 private:
  void RunInForeground(AsyncCompileJob* job) override {
    TRACE_COMPILE("(4) Async compilation failed.\n");
    // Fail compilation, invalidating the {AsyncCompileJob}.
    std::move(*job).Failed();
  }
};

AsyncStreamingProcessor::AsyncStreamingProcessor(AsyncCompileJob* job)
    : decoder_(job->enabled_features_, &job->detected_features_),
      job_(job),
      compilation_unit_builder_(nullptr) {}

// Process the module header.
bool AsyncStreamingProcessor::ProcessModuleHeader(
    base::Vector<const uint8_t> bytes) {
  TRACE_STREAMING("Process module header...\n");
  decoder_.DecodeModuleHeader(bytes);
  if (!decoder_.ok()) return false;
  // Note: We do not include the magic bytes in the hash; they are constant
  // anyways.
  return true;
}

// Process all sections except for the code section.
bool AsyncStreamingProcessor::ProcessSection(SectionCode section_code,
                                             base::Vector<const uint8_t> bytes,
                                             uint32_t offset) {
  TRACE_STREAMING("Process section %d ...\n", section_code);
  if (compilation_unit_builder_) {
    // We reached a section after the code section, we do not need the
    // compilation_unit_builder_ anymore.
    CommitCompilationUnits();
    compilation_unit_builder_.reset();
  }
  if (before_code_section_) {
    // Combine section hashes until code section.
    prefix_hasher_.AddRange(bytes);
  }
  if (section_code == SectionCode::kUnknownSectionCode) {
    size_t bytes_consumed = ModuleDecoder::IdentifyUnknownSection(
        &decoder_, bytes, offset, &section_code);
    if (!decoder_.ok()) return false;
    if (section_code == SectionCode::kUnknownSectionCode) {
      // Skip unknown sections that we do not know how to handle.
      return true;
    }
    // Remove the unknown section tag from the payload bytes.
    offset += bytes_consumed;
    bytes = bytes.SubVector(bytes_consumed, bytes.size());
  }
  decoder_.DecodeSection(section_code, bytes, offset);
  return decoder_.ok();
}

// Start the code section.
bool AsyncStreamingProcessor::ProcessCodeSectionHeader(
    int num_functions, uint32_t functions_mismatch_error_offset,
    std::shared_ptr<WireBytesStorage> wire_bytes_storage,
    size_t code_section_start, size_t code_section_length) {
  DCHECK_LE(0, code_section_length);
  before_code_section_ = false;
  TRACE_STREAMING("Start the code section with %d functions...\n",
                  num_functions);
  prefix_hasher_.Add(static_cast<uint32_t>(code_section_length));
  if (!decoder_.CheckFunctionsCount(static_cast<uint32_t>(num_functions),
                                    functions_mismatch_error_offset)) {
    return false;
  }

  DCHECK_LT(code_section_start, kMaxUInt32);
  DCHECK_LT(code_section_length, kMaxUInt32);
  decoder_.StartCodeSection({static_cast<uint32_t>(code_section_start),
                             static_cast<uint32_t>(code_section_length)});

  if (!GetWasmEngine()->GetStreamingCompilationOwnership(
          prefix_hasher_.hash(), job_->compile_imports_)) {
    // Known prefix, wait until the end of the stream and check the cache.
    prefix_cache_hit_ = true;
    return true;
  }

  // Execute the PrepareAndStartCompile step immediately and not in a separate
  // task.
  DCHECK_EQ(kWasmOrigin, decoder_.module()->origin);
  size_t code_size_estimate =
      wasm::WasmCodeManager::EstimateNativeModuleCodeSize(num_functions,
                                                          code_section_length);
  AsyncCompileJob::PrepareNativeModule{
      decoder_.shared_module(),
      // start_compilation: false; triggered when we receive the bodies.
      false,
      // lazy_functions_are_validated: false (bodies not received yet).
      false, code_size_estimate}
      .RunInBackground(job_);

  DCHECK_NOT_NULL(job_->new_native_module_);
  auto* compilation_state = Impl(job_->new_native_module_->compilation_state());
  compilation_state->SetWireBytesStorage(std::move(wire_bytes_storage));
  DCHECK_EQ(job_->new_native_module_->module()->origin, kWasmOrigin);

  // Set outstanding_finishers_ to 2, because both the AsyncCompileJob and the
  // AsyncStreamingProcessor have to finish.
  job_->outstanding_finishers_.store(2);
  // TODO(13209): Use PGO for streaming compilation, if available.
  constexpr ProfileInformation* kNoProfileInformation = nullptr;
  compilation_unit_builder_ = InitializeCompilation(
      job_->new_native_module_.get(), kNoProfileInformation);
  return true;
}

// Process a function body.
bool AsyncStreamingProcessor::ProcessFunctionBody(
    base::Vector<const uint8_t> bytes, uint32_t offset) {
  TRACE_STREAMING("Process function body %d ...\n", num_functions_);
  uint32_t func_index =
      decoder_.module()->num_imported_functions + num_functions_;
  ++num_functions_;
  // In case of {prefix_cache_hit} we still need the function body to be
  // decoded. Otherwise a later cache miss cannot be handled.
  decoder_.DecodeFunctionBody(func_index, static_cast<uint32_t>(bytes.size()),
                              offset);

  if (prefix_cache_hit_) {
    // Don't compile yet if we might have a cache hit.
    return true;
  }

  const WasmModule* module = decoder_.module();
  auto enabled_features = job_->enabled_features_;
  DCHECK_EQ(module->origin, kWasmOrigin);
  const bool lazy_module = v8_flags.wasm_lazy_compilation;
  CHECK_IMPLIES(v8_flags.wasm_jitless, !v8_flags.wasm_lazy_validation);
  bool validate_lazily_compiled_function =
      v8_flags.wasm_jitless ||
      (!v8_flags.wasm_lazy_validation && lazy_module &&
       !(v8_flags.experimental_wasm_compilation_hints &&
         module->compilation_priorities.contains(func_index)));
  if (validate_lazily_compiled_function) {
    // {bytes} is part of a section buffer owned by the streaming decoder. The
    // streaming decoder is held alive by the {AsyncCompileJob}, so we can just
    // use the {bytes} vector as long as the {AsyncCompileJob} is still running.
    if (!validate_functions_job_handle_) {
      validate_functions_job_data_.Initialize(module->num_declared_functions);
      validate_functions_job_handle_ = V8::GetCurrentPlatform()->CreateJob(
          TaskPriority::kUserVisible,
          std::make_unique<ValidateFunctionsStreamingJob>(
              module, enabled_features, &validate_functions_job_data_));
    }
    validate_functions_job_data_.AddUnit(func_index, bytes,
                                         validate_functions_job_handle_.get());
  }

  DCHECK_NOT_NULL(job_->new_native_module_);
  auto* compilation_state = Impl(job_->new_native_module_->compilation_state());
  compilation_state->InitializeCompilationUnitForSingleFunction(
      compilation_unit_builder_.get(), func_index);
  return true;
}

void AsyncStreamingProcessor::CommitCompilationUnits() {
  DCHECK(compilation_unit_builder_);
  compilation_unit_builder_->Commit();
}

void AsyncStreamingProcessor::OnFinishedChunk() {
  TRACE_STREAMING("FinishChunk...\n");
  if (compilation_unit_builder_) CommitCompilationUnits();
}

// Finish the processing of the stream.
void AsyncStreamingProcessor::OnFinishedStream(
    base::OwnedVector<const uint8_t> bytes, bool after_error) {
  TRACE_STREAMING("Finish stream...\n");
  ModuleResult module_result = decoder_.FinishDecoding();
  if (module_result.failed()) after_error = true;

  if (validate_functions_job_handle_) {
    // Wait for background validation to finish, then check if a validation
    // error was found.
    // TODO(13447): Do not block here; register validation as another finisher
    // instead.
    validate_functions_job_handle_->Join();
    validate_functions_job_handle_.reset();
    if (validate_functions_job_data_.found_error) after_error = true;
    job_->detected_features_ |=
        validate_functions_job_data_.detected_features.load(
            std::memory_order_relaxed);
  }

  job_->wire_bytes_ = ModuleWireBytes(bytes.as_vector());
  job_->bytes_copy_ = std::move(bytes);

  if (!after_error) {
    WasmDetectedFeatures detected_imports_features;
    if (WasmError error = ValidateAndSetBuiltinImports(
            module_result.value().get(), job_->wire_bytes_.module_bytes(),
            job_->compile_imports_, &detected_imports_features)) {
      after_error = true;
    } else {
      job_->detected_features_ |= detected_imports_features;
    }
  }

  // Record event metrics.
  v8::metrics::WasmModuleDecoded decode_event{
      .async = true,
      .streamed = true,
      .success = !after_error,
      .module_size_in_bytes = job_->wire_bytes_.length(),
      .function_count = static_cast<size_t>(num_functions_)};
  if (!job_->start_time_.IsNull()) {
    base::TimeDelta duration = base::TimeTicks::Now() - job_->start_time_;
    decode_event.wall_clock_duration_in_us = duration.InMicroseconds();
  }
  Isolate* isolate = job_->isolate_specific_info_.isolate_;
  isolate->metrics_recorder()->AddMainThreadEvent(
      decode_event, job_->isolate_specific_info_.context_id_);

  if (after_error) {
    if (job_->new_native_module_ &&
        job_->new_native_module_->wire_bytes().empty()) {
      // Clean up the temporary cache entry.
      GetWasmEngine()->StreamingCompilationFailed(prefix_hasher_.hash(),
                                                  job_->compile_imports_);
    }
    // Calling {Failed} will invalidate the {AsyncCompileJob} and delete {this}.
    std::move(*job_).Failed();
    return;
  }

  std::shared_ptr<WasmModule> module = std::move(module_result).value();

  // At this point we identified the module as valid (except maybe for function
  // bodies, if lazy validation is enabled).
  // This DCHECK could be considered slow, but it only happens once per async
  // module compilation, and we only re-decode the module structure, without
  // validating function bodies. Overall this does not add a lot of overhead.
#ifdef DEBUG
  WasmDetectedFeatures detected_module_features;
  DCHECK(DecodeWasmModule(job_->enabled_features_,
                          job_->bytes_copy_.as_vector(),
                          /* validate functions */ false, kWasmOrigin,
                          &detected_module_features)
             .ok());
  // Module decoding should not detect any new features.
  DCHECK(job_->detected_features_.contains_all(detected_module_features));
#endif

  DCHECK_EQ(NativeModuleCache::PrefixHash(job_->wire_bytes_.module_bytes()),
            prefix_hasher_.hash());
  if (prefix_cache_hit_) {
    // Restart as an asynchronous, non-streaming compilation. Most likely
    // {PrepareNativeModule} will get the native module from the cache.
    size_t code_size_estimate =
        wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module.get());
    job_->DoAsync<AsyncCompileJob::PrepareNativeModule>(
        std::move(module), true /* start_compilation */,
        false /* lazy_functions_are_validated_ */, code_size_estimate);
    return;
  }

  // We have to open a HandleScope and prepare the Context for
  // CreateNativeModule, PrepareRuntimeObjects and FinishCompile as this is a
  // callback from the embedder.
  HandleScope scope(isolate);
  SaveAndSwitchContext saved_context(
      isolate, *job_->isolate_specific_info_.native_context_);

  // Record the size of the wire bytes and the number of functions. In
  // synchronous and asynchronous (non-streaming) compilation, this happens in
  // {DecodeWasmModule}.
  auto* module_size_histogram =
      isolate->counters()->wasm_wasm_module_size_bytes();
  module_size_histogram->AddSample(job_->wire_bytes_.module_bytes().length());
  auto* num_functions_histogram =
      isolate->counters()->wasm_functions_per_wasm_module();
  num_functions_histogram->AddSample(static_cast<int>(num_functions_));

  std::shared_ptr<NativeModule> final_native_module = job_->new_native_module_;
  bool cache_hit = false;
  if (!final_native_module) {
    // We are processing a WebAssembly module without code section. Create the
    // native module now (would otherwise happen in {PrepareNativeModule} or
    // {ProcessCodeSectionHeader}).
    constexpr size_t kCodeSizeEstimate = 0;
    std::tie(final_native_module, cache_hit) =
        job_->GetOrCreateNativeModule(std::move(module), kCodeSizeEstimate);
  } else {
    final_native_module->SetWireBytes(std::move(job_->bytes_copy_));
  }

  // Only decrement the finisher count *after* setting the wire bytes. The
  // native module needs to be ready to use when the last finisher is done.
  const bool needs_finish = job_->DecrementAndCheckFinisherCount();
  if (!cache_hit && needs_finish) {
    const bool failed = job_->new_native_module_->compilation_state()->failed();
    final_native_module = GetWasmEngine()->UpdateNativeModuleCache(
        failed, job_->new_native_module_, isolate);
    cache_hit = final_native_module != job_->new_native_module_;
    // We finally call {Failed} or {FinishCompile}, which will invalidate the
    // {AsyncCompileJob} and delete {this}.
    if (failed) {
      std::move(*job_).Failed();
      return;
    }
  }

  if (needs_finish) {
    std::move(*job_).FinishCompile(std::move(final_native_module), {},
                                   cache_hit);
  }
}

void AsyncStreamingProcessor::OnAbort() {
  TRACE_STREAMING("Abort stream...\n");
  if (validate_functions_job_handle_) {
    validate_functions_job_handle_->Cancel();
    validate_functions_job_handle_.reset();
  }
  if (job_->new_native_module_ &&
      job_->new_native_module_->wire_bytes().empty()) {
    // Clean up the temporary cache entry.
    GetWasmEngine()->StreamingCompilationFailed(prefix_hasher_.hash(),
                                                job_->compile_imports_);
  }
  // {Abort} invalidates the {AsyncCompileJob}, which in turn deletes {this}.
  job_->Abort();
}

bool AsyncStreamingProcessor::Deserialize(
    base::Vector<const uint8_t> module_bytes,
    base::OwnedVector<const uint8_t>& wire_bytes) {
  TRACE_EVENT0("v8.wasm", "wasm.Deserialize");
  Isolate* isolate = job_->isolate_specific_info_.isolate_;
  std::optional<TimedHistogramScope> time_scope;
  if (base::TimeTicks::IsHighResolution()) {
    time_scope.emplace(isolate->counters()->wasm_deserialization_time(),
                       isolate);
  }
  // DeserializeNativeModule and FinishCompile assume that they are executed in
  // a HandleScope, and that a context is set on the isolate.
  HandleScope scope(isolate);
  SaveAndSwitchContext saved_context(
      isolate, *job_->isolate_specific_info_.native_context_);

  // TODO(clemensb): `DeserializeNativeModule` should actually return a
  // `shared_ptr<NativeModule>`.
  DirectHandle<WasmModuleObject> module_object;
  if (!DeserializeNativeModule(isolate, job_->enabled_features_, module_bytes,
                               wire_bytes, job_->compile_imports_,
                               base::VectorOf(job_->stream_->url()))
           .ToHandle(&module_object)) {
    return false;
  }

  DCHECK_NULL(job_->new_native_module_);
  std::shared_ptr<NativeModule> deserialized_native_module =
      module_object->shared_native_module();
  job_->wire_bytes_ = ModuleWireBytes(deserialized_native_module->wire_bytes());
  // Calling {FinishCompile} deletes the {AsyncCompileJob} and {this}.
  std::move(*job_).FinishCompile(deserialized_native_module, module_object,
                                 false);
  return true;
}

CompilationStateImpl::CompilationStateImpl(
    const std::shared_ptr<NativeModule>& native_module,
    WasmDetectedFeatures detected_features)
    : native_module_(native_module.get()),
      native_module_weak_(native_module),
      compilation_unit_queues_(native_module->num_imported_functions(),
                               native_module->num_declared_functions()),
      detected_features_(detected_features) {}

void CompilationStateImpl::InitCompileJob() {
  DCHECK_NULL(baseline_compile_job_);
  DCHECK_NULL(top_tier_compile_job_);
  // Create the job, but don't spawn workers yet. This will happen on
  // {NotifyConcurrencyIncrease}.
  baseline_compile_job_ = V8::GetCurrentPlatform()->CreateJob(
      TaskPriority::kUserVisible,
      std::make_unique<BackgroundCompileJob>(native_module_weak_,
                                             CompilationTier::kBaseline));
  top_tier_compile_job_ = V8::GetCurrentPlatform()->CreateJob(
      TaskPriority::kUserVisible,
      std::make_unique<BackgroundCompileJob>(native_module_weak_,
                                             CompilationTier::kTopTier));
}

void CompilationStateImpl::CancelCompilation(
    CompilationStateImpl::CancellationPolicy cancellation_policy) {
  base::MutexGuard callbacks_guard(&callbacks_mutex_);

  if (cancellation_policy == kCancelInitialCompilation &&
      finished_events_.contains(
          CompilationEvent::kFinishedBaselineCompilation)) {
    // Initial compilation already finished; cannot be cancelled.
    return;
  }

  // std::memory_order_relaxed is sufficient because no other state is
  // synchronized with |compile_cancelled_|.
  compile_cancelled_.store(true, std::memory_order_relaxed);

  // No more callbacks after abort.
  callbacks_.clear();
}

bool CompilationStateImpl::cancelled() const {
  return compile_cancelled_.load(std::memory_order_relaxed);
}

void CompilationStateImpl::ApplyEagerTierUpToInitialProgress(size_t hint_idx) {
  // Get old information.
  uint8_t& progress = compilation_progress_[hint_idx];
  ExecutionTier old_baseline_tier = RequiredBaselineTierField::decode(progress);

  // Compute new information.
  ExecutionTier new_baseline_tier = ExecutionTier::kTurbofan;
  ExecutionTier new_top_tier = ExecutionTier::kTurbofan;

  progress = RequiredBaselineTierField::update(progress, new_baseline_tier);
  progress = RequiredTopTierField::update(progress, new_top_tier);

  // Update counter for outstanding baseline units.
  outstanding_baseline_units_ +=
      1 - (old_baseline_tier != ExecutionTier::kNone);
}

void CompilationStateImpl::ApplyCompilationPriorityToInitialProgress(
    size_t hint_idx, CompilationPriority priority) {
  // Get old information.
  uint8_t& progress = compilation_progress_[hint_idx];
  ExecutionTier old_baseline_tier = RequiredBaselineTierField::decode(progress);

  // Compute new information.
  // Always compile eagerly (blockingly) with Liftoff if a compilation-priority
  // hint is present. Then, if optimization_priority is present and the function
  // is not only executed once, compile with Turbofan in the background.
  bool optimization_priority_not_finite =
      priority.optimization_priority >=
          kOptimizationPriorityExecutedOnceSentinel ||
      priority.optimization_priority ==
          kOptimizationPriorityNotSpecifiedSentinel;
  ExecutionTier new_baseline_tier =
      v8_flags.liftoff ? ExecutionTier::kLiftoff : ExecutionTier::kTurbofan;
  ExecutionTier new_top_tier = (optimization_priority_not_finite ||
                                !v8_flags.liftoff || v8_flags.liftoff_only)
                                   ? ExecutionTier::kNone
                                   : ExecutionTier::kTurbofan;

  progress = RequiredBaselineTierField::update(progress, new_baseline_tier);
  progress = RequiredTopTierField::update(progress, new_top_tier);

  // Update counter for outstanding baseline units.
  outstanding_baseline_units_ +=
      1 - (old_baseline_tier != ExecutionTier::kNone);
}

void CompilationStateImpl::ApplyPgoInfoToInitialProgress(
    ProfileInformation* pgo_info) {
  // Functions that were executed in the profiling run are eagerly compiled to
  // Liftoff.
  const WasmModule* module = native_module_->module();
  for (int func_index : pgo_info->executed_functions()) {
    uint8_t& progress =
        compilation_progress_[declared_function_index(module, func_index)];
    ExecutionTier old_baseline_tier =
        RequiredBaselineTierField::decode(progress);
    // If the function is already marked for eager compilation, we are good.
    if (old_baseline_tier != ExecutionTier::kNone) continue;

    // Set the baseline tier to Liftoff, so we eagerly compile to Liftoff.
    // TODO(13288): Compile Liftoff code in the background, if lazy compilation
    // is enabled.
    progress =
        RequiredBaselineTierField::update(progress, ExecutionTier::kLiftoff);
    ++outstanding_baseline_units_;
  }

  // Functions that were tiered up during PGO generation are eagerly compiled to
  // TurboFan (in the background, not blocking instantiation).
  for (int func_index : pgo_info->tiered_up_functions()) {
    uint8_t& progress =
        compilation_progress_[declared_function_index(module, func_index)];
    ExecutionTier old_baseline_tier =
        RequiredBaselineTierField::decode(progress);
    ExecutionTier old_top_tier = RequiredTopTierField::decode(progress);
    // If the function is already marked for eager or background compilation to
    // TurboFan, we are good.
    if (old_baseline_tier == ExecutionTier::kTurbofan) continue;
    if (old_top_tier == ExecutionTier::kTurbofan) continue;

    // Set top tier to TurboFan, so we eagerly trigger compilation in the
    // background.
    progress = RequiredTopTierField::update(progress, ExecutionTier::kTurbofan);
  }
}

void CompilationStateImpl::ApplyPgoInfoLate(ProfileInformation* pgo_info) {
  TRACE_EVENT0("v8.wasm", "wasm.ApplyPgoInfo");
  const WasmModule* module = native_module_->module();
  CompilationUnitBuilder builder{native_module_};

  base::MutexGuard guard(&callbacks_mutex_);
  // Functions that were executed in the profiling run are eagerly compiled to
  // Liftoff (in the background).
  for (int func_index : pgo_info->executed_functions()) {
    uint8_t& progress =
        compilation_progress_[declared_function_index(module, func_index)];
    ExecutionTier old_baseline_tier =
        RequiredBaselineTierField::decode(progress);
    // If the function is already marked for eager compilation, we are good.
    if (old_baseline_tier != ExecutionTier::kNone) continue;

    // If we already compiled Liftoff or TurboFan code, we are also good.
    ExecutionTier reached_tier = ReachedTierField::decode(progress);
    if (reached_tier >= ExecutionTier::kLiftoff) continue;

    // Set the baseline tier to Liftoff and schedule a compilation unit.
    progress =
        RequiredBaselineTierField::update(progress, ExecutionTier::kLiftoff);
    // Add this as a "top tier unit" since it does not contribute to initial
    // compilation ("baseline finished" might already be triggered).
    // TODO(clemensb): Rename "baseline finished" to "initial compile finished".
    // TODO(clemensb): Avoid scheduling both a Liftoff and a TurboFan unit, or
    // prioritize Liftoff when executing the units.
    builder.AddTopTierUnit(func_index, ExecutionTier::kLiftoff);
  }

  // Functions that were tiered up during PGO generation are eagerly compiled to
  // TurboFan in the background.
  for (int func_index : pgo_info->tiered_up_functions()) {
    uint8_t& progress =
        compilation_progress_[declared_function_index(module, func_index)];
    ExecutionTier old_baseline_tier =
        RequiredBaselineTierField::decode(progress);
    ExecutionTier old_top_tier = RequiredTopTierField::decode(progress);
    // If the function is already marked for eager or background compilation to
    // TurboFan, we are good.
    if (old_baseline_tier == ExecutionTier::kTurbofan) continue;
    if (old_top_tier == ExecutionTier::kTurbofan) continue;

    // If we already compiled TurboFan code, we are also good.
    ExecutionTier reached_tier = ReachedTierField::decode(progress);
    if (reached_tier == ExecutionTier::kTurbofan) continue;

    // Set top tier to TurboFan and schedule a compilation unit.
    progress = RequiredTopTierField::update(progress, ExecutionTier::kTurbofan);
    builder.AddTopTierUnit(func_index, ExecutionTier::kTurbofan);
  }
  builder.Commit();
}

void CompilationStateImpl::InitializeCompilationProgress(
    ProfileInformation* pgo_info) {
  DCHECK(!failed());

  base::MutexGuard guard(&callbacks_mutex_);

  if (!v8_flags.wasm_jitless) {
    auto* module = native_module_->module();

    DCHECK_EQ(0, outstanding_baseline_units_);

    // Compute the default compilation progress for all functions, and set it.
    const ExecutionTierPair default_tiers = GetDefaultTiersPerModule(
        native_module_, native_module_->IsInDebugState(), IsLazyModule(module));
    const uint8_t default_progress =
        RequiredBaselineTierField::encode(default_tiers.baseline_tier) |
        RequiredTopTierField::encode(default_tiers.top_tier) |
        ReachedTierField::encode(ExecutionTier::kNone);
    DCHECK_NULL(compilation_progress_);
    compilation_progress_ = base::OwnedVector<uint8_t>::New(
        module->num_declared_functions, default_progress);
    if (default_tiers.baseline_tier != ExecutionTier::kNone) {
      outstanding_baseline_units_ += module->num_declared_functions;
    }

    if (native_module_->enabled_features().has_compilation_hints()) {
      for (auto [function_index, compilation_priority] :
           module->compilation_priorities) {
        // If an imported function has a compilation hint, ignore it.
        if (function_index < module->num_imported_functions) continue;
        uint32_t declared_function_index =
            function_index - module->num_imported_functions;
        // If a function index is out of bounds, ignore it.
        if (declared_function_index >= module->num_declared_functions) {
          continue;
        }
        ApplyCompilationPriorityToInitialProgress(declared_function_index,
                                                  compilation_priority);
      }
    }

    // Apply --wasm-eager-tier-up-function, if given.
    if (V8_UNLIKELY(
            v8_flags.wasm_eager_tier_up_function >= 0 &&
            static_cast<uint32_t>(v8_flags.wasm_eager_tier_up_function) >=
                module->num_imported_functions &&
            static_cast<uint32_t>(v8_flags.wasm_eager_tier_up_function) <
                module->functions.size())) {
      uint32_t func_idx =
          v8_flags.wasm_eager_tier_up_function - module->num_imported_functions;
      ApplyEagerTierUpToInitialProgress(func_idx);
    }
  }

  // Apply PGO information, if available.
  if (pgo_info) ApplyPgoInfoToInitialProgress(pgo_info);

  // Trigger callbacks if module needs no baseline or top tier compilation. This
  // can be the case for an empty or fully lazy module.
  TriggerOutstandingCallbacks();
}

void CompilationStateImpl::InitializeCompilationUnitForSingleFunction(
    CompilationUnitBuilder* builder, int function_index) {
  if (v8_flags.wasm_jitless) return;

  // Only during streaming compilation, we reach here without holding
  // `callbacks_mutex_`.
  // This avoids the observed overhead of taking the lock once per function.
  // This is correct (data-race free), because the only mutation of
  // `compilation_progress_` before reading it here without holding the lock is
  // when initializing the compilation during parsing of the code section
  // header (see `InitializeCompilationProgress` and
  // `AsyncStreamingProcessor::ProcessCodeSectionHeader`), which must happen
  // before processing functions.
  // Also, this read here happens exactly once per function, only for the
  // initial compilation, and all accesses to `compilation_progress_` thereafter
  // are synchronized via `callbacks_mutex_`.
  uint8_t function_progress = compilation_progress_[declared_function_index(
      native_module_->module(), function_index)];
  ExecutionTier required_baseline_tier =
      CompilationStateImpl::RequiredBaselineTierField::decode(
          function_progress);
  ExecutionTier required_top_tier =
      CompilationStateImpl::RequiredTopTierField::decode(function_progress);
  ExecutionTier reached_tier =
      CompilationStateImpl::ReachedTierField::decode(function_progress);

  if (reached_tier < required_baseline_tier) {
    builder->AddBaselineUnit(function_index, required_baseline_tier);
  }
  if (reached_tier < required_top_tier &&
      required_baseline_tier != required_top_tier) {
    builder->AddTopTierUnit(function_index, required_top_tier);
  }
}

void CompilationStateImpl::InitializeCompilationUnits(
    std::unique_ptr<CompilationUnitBuilder> builder) {
  if (v8_flags.wasm_jitless) return;

  {
    base::MutexGuard guard(&callbacks_mutex_);
    const WasmModule* module = native_module_->module();
    DCHECK_EQ(module->num_declared_functions, compilation_progress_.size());
    int start = module->num_imported_functions;
    int end = start + module->num_declared_functions;
    for (int func_index = start; func_index < end; ++func_index) {
      InitializeCompilationUnitForSingleFunction(builder.get(), func_index);
    }
  }
  builder->Commit();
}

void CompilationStateImpl::InitializeCompilationProgressAfterDeserialization(
    base::Vector<const int> lazy_functions,
    base::Vector<const int> eager_functions) {
  TRACE_EVENT2("v8.wasm", "wasm.CompilationAfterDeserialization",
               "num_lazy_functions", lazy_functions.size(),
               "num_eager_functions", eager_functions.size());
  base::ElapsedTimer lazy_compile_time;
  if (base::TimeTicks::IsHighResolution()) {
    lazy_compile_time.Start();
  }

  auto* module = native_module_->module();
  {
    base::MutexGuard guard(&callbacks_mutex_);
    DCHECK_NULL(compilation_progress_);

    // Initialize the compilation progress as if everything was
    // TurboFan-compiled.
    constexpr uint8_t kProgressAfterTurbofanDeserialization =
        RequiredBaselineTierField::encode(ExecutionTier::kLiftoff) |
        RequiredTopTierField::encode(ExecutionTier::kTurbofan) |
        ReachedTierField::encode(ExecutionTier::kTurbofan);
    compilation_progress_ = base::OwnedVector<uint8_t>::New(
        module->num_declared_functions, kProgressAfterTurbofanDeserialization);

    // Update compilation state for lazy functions.
    constexpr uint8_t kProgressForLazyFunctions =
        RequiredBaselineTierField::encode(ExecutionTier::kNone) |
        RequiredTopTierField::encode(ExecutionTier::kNone) |
        ReachedTierField::encode(ExecutionTier::kNone);
    for (auto func_index : lazy_functions) {
      compilation_progress_[declared_function_index(module, func_index)] =
          kProgressForLazyFunctions;
    }

    // Update compilation state for eagerly compiled functions.
    constexpr bool kNotLazy = false;
    ExecutionTierPair default_tiers = GetDefaultTiersPerModule(
        native_module_, native_module_->IsInDebugState(), kNotLazy);
    uint8_t progress_for_eager_functions =
        RequiredBaselineTierField::encode(default_tiers.baseline_tier) |
        RequiredTopTierField::encode(default_tiers.top_tier) |
        ReachedTierField::encode(ExecutionTier::kNone);
    for (auto func_index : eager_functions) {
      // Check that {func_index} is not contained in {lazy_functions}.
      DCHECK_EQ(
          compilation_progress_[declared_function_index(module, func_index)],
          kProgressAfterTurbofanDeserialization);
      compilation_progress_[declared_function_index(module, func_index)] =
          progress_for_eager_functions;
    }
    DCHECK_NE(ExecutionTier::kNone, default_tiers.baseline_tier);
    outstanding_baseline_units_ += eager_functions.size();

    // Baseline compilation is done if we do not have any Liftoff functions to
    // compile.
    if (eager_functions.empty() || v8_flags.wasm_lazy_compilation) {
      finished_events_.Add(CompilationEvent::kFinishedBaselineCompilation);
    }
  }
  auto builder = std::make_unique<CompilationUnitBuilder>(native_module_);
  InitializeCompilationUnits(std::move(builder));
  if (!v8_flags.wasm_lazy_compilation) {
    WaitForBaselineCompileJob();
  }

  if (lazy_compile_time.IsStarted()) {
    native_module_->counter_updates()->AddTimedSample(
        &Counters::wasm_compile_after_deserialize, lazy_compile_time.Elapsed());
  }
}

void CompilationStateImpl::AddCallback(
    std::unique_ptr<CompilationEventCallback> callback) {
  base::MutexGuard callbacks_guard(&callbacks_mutex_);
  // Immediately trigger events that already happened.
  for (auto event : {CompilationEvent::kFinishedBaselineCompilation,
                     CompilationEvent::kFailedCompilation}) {
    if (finished_events_.contains(event)) {
      callback->call(event);
    }
  }
  constexpr base::EnumSet<CompilationEvent> kFinalEvents{
      CompilationEvent::kFailedCompilation};
  if (!finished_events_.contains_any(kFinalEvents)) {
    callbacks_.emplace_back(std::move(callback));
  }
}

void CompilationStateImpl::CommitCompilationUnits(
    base::Vector<WasmCompilationUnit> baseline_units,
    base::Vector<WasmCompilationUnit> top_tier_units) {
  base::MutexGuard guard{&mutex_};
  if (!baseline_units.empty() || !top_tier_units.empty()) {
    compilation_unit_queues_.AddUnits(baseline_units, top_tier_units,
                                      native_module_->module());
  }
  if (!baseline_units.empty()) {
    DCHECK(baseline_compile_job_->IsValid());
    baseline_compile_job_->NotifyConcurrencyIncrease();
  }
  if (!top_tier_units.empty()) {
    DCHECK(top_tier_compile_job_->IsValid());
    top_tier_compile_job_->NotifyConcurrencyIncrease();
  }
}

void CompilationStateImpl::CommitTopTierCompilationUnit(
    WasmCompilationUnit unit) {
  CommitCompilationUnits({}, {&unit, 1});
}

void CompilationStateImpl::AddTopTierPriorityCompilationUnit(
    WasmCompilationUnit unit, size_t priority) {
  compilation_unit_queues_.AddTopTierPriorityUnit(unit, priority);
  // We should not have a {CodeSpaceWriteScope} open at this point, as
  // {NotifyConcurrencyIncrease} can spawn new threads which could inherit PKU
  // permissions (which would be a security issue).
  top_tier_compile_job_->NotifyConcurrencyIncrease();
}

CompilationUnitQueues::Queue* CompilationStateImpl::GetQueueForCompileTask(
    int task_id) {
  return compilation_unit_queues_.GetQueueForTask(task_id);
}

std::optional<WasmCompilationUnit> CompilationStateImpl::GetNextCompilationUnit(
    CompilationUnitQueues::Queue* queue, CompilationTier tier) {
  return compilation_unit_queues_.GetNextUnit(queue, tier);
}

void CompilationStateImpl::OnFinishedUnits(
    base::Vector<WasmCode*> code_vector) {
  TRACE_EVENT1(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
               "wasm.OnFinishedUnits", "units", code_vector.size());

  base::MutexGuard guard(&callbacks_mutex_);

  // Assume an order of execution tiers that represents the quality of their
  // generated code.
  static_assert(ExecutionTier::kNone < ExecutionTier::kLiftoff &&
                    ExecutionTier::kLiftoff < ExecutionTier::kTurbofan,
                "Assume an order on execution tiers");

  if (!v8_flags.wasm_jitless) {
    DCHECK_EQ(compilation_progress_.size(),
              native_module_->module()->num_declared_functions);
  }

  bool has_top_tier_code = false;

  for (size_t i = 0; i < code_vector.size(); i++) {
    WasmCode* code = code_vector[i];
    DCHECK_NOT_NULL(code);
    DCHECK_LT(code->index(), native_module_->num_functions());

    has_top_tier_code |= code->tier() == ExecutionTier::kTurbofan;

    if (code->index() <
        static_cast<int>(native_module_->num_imported_functions())) {
      // Import wrapper.
      DCHECK_EQ(code->tier(), ExecutionTier::kTurbofan);
      outstanding_baseline_units_--;
    } else {
      // Function.
      DCHECK_NE(code->tier(), ExecutionTier::kNone);

      // Read function's compilation progress.
      // This view on the compilation progress may differ from the actually
      // compiled code. Any lazily compiled function does not contribute to the
      // compilation progress but may publish code to the code manager.
      uint8_t& function_progress =
          compilation_progress_[declared_function_index(
              native_module_->module(), code->index())];
      ExecutionTier required_baseline_tier =
          RequiredBaselineTierField::decode(function_progress);
      ExecutionTier reached_tier = ReachedTierField::decode(function_progress);

      // Check whether required baseline or top tier are reached.
      if (reached_tier < required_baseline_tier &&
          required_baseline_tier <= code->tier()) {
        DCHECK_GT(outstanding_baseline_units_, 0);
        outstanding_baseline_units_--;
      }
      if (code->tier() == ExecutionTier::kTurbofan) {
        bytes_since_last_chunk_ += code->instructions().size();
      }

      // Update function's compilation progress.
      if (code->tier() > reached_tier) {
        function_progress =
            ReachedTierField::update(function_progress, code->tier());
      }
      // Allow another top tier compilation if deopts are enabled and the
      // currently installed code object is a Liftoff object.
      // Ideally, this would be done only if the code->tier() ==
      // ExecutionTier::Liftoff as the code object for which we run this
      // function should be the same as the one installed on the native_module.
      // This is unfortunately not the case as installing a code object on the
      // native module and updating the compilation_progress_ and the
      // CompilationUnitQueues::top_tier_compiled_ are not synchronized.
      // Note: GetCode() acquires the NativeModule::allocation_mutex_, so this
      // could cause deadlocks if any other place acquires
      // NativeModule::allocation_mutex_ first and then
      // CompilationStateImpl::callbacks_mutex_!
      const bool is_liftoff = code->tier() == ExecutionTier::kLiftoff;
      auto published_code_is_liftoff = [this](int index) {
        WasmCode* code = native_module_->GetCode(index);
        if (code == nullptr) return false;
        return code->is_liftoff();
      };
      if (v8_flags.wasm_deopt &&
          (is_liftoff || published_code_is_liftoff(code->index()))) {
        // Setting the reached tier below the baseline tier would create an
        // inconsistent state and has actually led to crashes before (see
        // https://crbug.com/379086474).
        DCHECK_LE(required_baseline_tier, ExecutionTier::kLiftoff);
        function_progress = ReachedTierField::update(function_progress,
                                                     ExecutionTier::kLiftoff);
        compilation_unit_queues_.AllowAnotherTopTierJob(code->index());
      }
      DCHECK_LE(0, outstanding_baseline_units_);
    }
  }

  // Update the {last_top_tier_compilation_timestamp_} if it is set (i.e. a
  // delayed task has already been spawned).
  if (has_top_tier_code && !last_top_tier_compilation_timestamp_.IsNull()) {
    last_top_tier_compilation_timestamp_ = base::TimeTicks::Now();
  }

  TriggerOutstandingCallbacks();
}

namespace {
class TriggerCodeCachingAfterTimeoutTask : public v8::Task {
 public:
  explicit TriggerCodeCachingAfterTimeoutTask(
      std::weak_ptr<NativeModule> native_module)
      : native_module_(std::move(native_module)) {}

  void Run() override {
    if (std::shared_ptr<NativeModule> native_module = native_module_.lock()) {
      Impl(native_module->compilation_state())->TriggerCachingAfterTimeout();
    }
  }

 private:
  const std::weak_ptr<NativeModule> native_module_;
};
}  // namespace

void CompilationStateImpl::TriggerOutstandingCallbacks() {
  callbacks_mutex_.AssertHeld();

  base::EnumSet<CompilationEvent> triggered_events;
  if (outstanding_baseline_units_ == 0) {
    triggered_events.Add(CompilationEvent::kFinishedBaselineCompilation);
  }

  // For dynamic tiering, trigger "compilation chunk finished" after a new chunk
  // of size {v8_flags.wasm_caching_threshold}.
  if (v8_flags.wasm_dynamic_tiering &&
      static_cast<size_t>(v8_flags.wasm_caching_threshold) <=
          bytes_since_last_chunk_) {
    // Trigger caching immediately if
    // - there is no timeout,
    // - the hard threshold was reached, or
    // - we are running single-threaded.
    if (v8_flags.single_threaded || v8_flags.wasm_caching_timeout_ms <= 0 ||
        static_cast<size_t>(v8_flags.wasm_caching_hard_threshold) <=
            bytes_since_last_chunk_) {
      triggered_events.Add(CompilationEvent::kFinishedCompilationChunk);
      bytes_since_last_chunk_ = 0;
    } else if (last_top_tier_compilation_timestamp_.IsNull()) {
      // Trigger a task after the given timeout; that task will only trigger
      // caching if no new code was added until then. Otherwise, it will
      // re-schedule itself.
      V8::GetCurrentPlatform()->PostDelayedTaskOnWorkerThread(
          TaskPriority::kUserVisible,
          std::make_unique<TriggerCodeCachingAfterTimeoutTask>(
              native_module_weak_),
          1e-3 * v8_flags.wasm_caching_timeout_ms);

      // Set the timestamp (will be updated by {OnFinishedUnits} if more
      // top-tier compilation finished before the delayed task is being run).
      last_top_tier_compilation_timestamp_ = base::TimeTicks::Now();
    }
  }

  if (compile_failed_.load(std::memory_order_relaxed)) {
    // *Only* trigger the "failed" event.
    triggered_events =
        base::EnumSet<CompilationEvent>({CompilationEvent::kFailedCompilation});
  }

  TriggerCallbacks(triggered_events);
}

void CompilationStateImpl::TriggerCallbacks(
    base::EnumSet<CompilationEvent> events) {
  if (events.empty()) return;

  // Don't trigger past events again.
  events -= finished_events_;
  // There can be multiple compilation chunks, thus do not store this.
  finished_events_ |= events - CompilationEvent::kFinishedCompilationChunk;

  for (auto event :
       {std::make_pair(CompilationEvent::kFailedCompilation,
                       "wasm.CompilationFailed"),
        std::make_pair(CompilationEvent::kFinishedBaselineCompilation,
                       "wasm.BaselineFinished"),
        std::make_pair(CompilationEvent::kFinishedCompilationChunk,
                       "wasm.CompilationChunkFinished")}) {
    if (!events.contains(event.first)) continue;
    DCHECK_NE(compilation_id_, kInvalidCompilationID);
    TRACE_EVENT1("v8.wasm", event.second, "id", compilation_id_);
    for (auto& callback : callbacks_) {
      callback->call(event.first);
    }
  }

  if (outstanding_baseline_units_ == 0) {
    auto new_end = std::remove_if(
        callbacks_.begin(), callbacks_.end(), [](const auto& callback) {
          return callback->release_after_final_event();
        });
    callbacks_.erase(new_end, callbacks_.end());
  }
}

void CompilationStateImpl::TriggerCachingAfterTimeout() {
  base::MutexGuard guard{&callbacks_mutex_};

  // It can happen that we reached the hard threshold while waiting for the
  // timeout to expire. In that case, {bytes_since_last_chunk_} might be zero
  // and there is nothing new to cache.
  if (bytes_since_last_chunk_ == 0) return;

  DCHECK(!last_top_tier_compilation_timestamp_.IsNull());
  base::TimeTicks caching_time =
      last_top_tier_compilation_timestamp_ +
      base::TimeDelta::FromMilliseconds(v8_flags.wasm_caching_timeout_ms);
  base::TimeDelta time_until_caching = caching_time - base::TimeTicks::Now();
  // If we are still half a millisecond or more away from the timeout,
  // reschedule the task. Otherwise, call the caching callback.
  if (time_until_caching >= base::TimeDelta::FromMicroseconds(500)) {
    int ms_remaining =
        static_cast<int>(time_until_caching.InMillisecondsRoundedUp());
    DCHECK_LE(1, ms_remaining);
    V8::GetCurrentPlatform()->PostDelayedTaskOnWorkerThread(
        TaskPriority::kUserVisible,
        std::make_unique<TriggerCodeCachingAfterTimeoutTask>(
            native_module_weak_),
        ms_remaining);
    return;
  }

  TriggerCallbacks({CompilationEvent::kFinishedCompilationChunk});
  last_top_tier_compilation_timestamp_ = {};
  bytes_since_last_chunk_ = 0;
}

void CompilationStateImpl::OnCompilationStopped(
    WasmDetectedFeatures detected_features) {
  WasmDetectedFeatures new_detected_features =
      UpdateDetectedFeatures(detected_features);
  if (new_detected_features.empty()) return;

  // New detected features can only happen during eager compilation or if lazy
  // validation is enabled. Compilation hints enables eager compilation if there
  // are compilation-priority hints in the module, so it should be included
  // here.
  // The exceptions are currently stringref and imported strings, which are only
  // detected on top-tier compilation.
  DCHECK(!v8_flags.wasm_lazy_compilation || v8_flags.wasm_lazy_validation ||
         (v8_flags.experimental_wasm_compilation_hints &&
          !native_module_->module()->compilation_priorities.empty()) ||
         (new_detected_features -
          WasmDetectedFeatures{{WasmDetectedFeature::stringref,
                                WasmDetectedFeature::imported_strings_utf8,
                                WasmDetectedFeature::imported_strings}})
             .empty());
  // TODO(clemensb): Fix reporting of late detected features (relevant for lazy
  // validation and for stringref).
}

WasmDetectedFeatures CompilationStateImpl::UpdateDetectedFeatures(
    WasmDetectedFeatures detected_features) {
  WasmDetectedFeatures old_features =
      detected_features_.load(std::memory_order_relaxed);
  while (!detected_features_.compare_exchange_weak(
      old_features, old_features | detected_features,
      std::memory_order_relaxed)) {
    // Retry with updated {old_features}.
  }
  return detected_features - old_features;
}

void CompilationStateImpl::PublishCompilationResults(
    std::vector<UnpublishedWasmCode> unpublished_code) {
  if (unpublished_code.empty()) return;

#if DEBUG
  // We don't compile import wrappers eagerly.
  for (const auto& [code, assumptions] : unpublished_code) {
    int func_index = code->index();
    DCHECK_LE(native_module_->num_imported_functions(), func_index);
    DCHECK_LT(func_index, native_module_->num_functions());
  }
#endif
  PublishCode(base::VectorOf(unpublished_code));
}

std::vector<WasmCode*> CompilationStateImpl::PublishCode(
    base::Vector<UnpublishedWasmCode> code) {
  WasmCodeRefScope code_ref_scope;
  std::vector<WasmCode*> published_code =
      native_module_->PublishCode(std::move(code));
  // Defer logging code in case wire bytes were not fully received yet.
  if (native_module_->log_code() && native_module_->HasWireBytes()) {
    GetWasmEngine()->LogCode(base::VectorOf(published_code));
  }

  OnFinishedUnits(base::VectorOf(published_code));
  return published_code;
}

void CompilationStateImpl::SchedulePublishCompilationResults(
    std::vector<UnpublishedWasmCode> unpublished_code, CompilationTier tier) {
  PublishState& state = publish_state_[tier];
  {
    base::MutexGuard guard(&state.mutex_);
    if (state.publisher_running_) {
      // Add new code to the queue and return.
      state.publish_queue_.reserve(state.publish_queue_.size() +
                                   unpublished_code.size());
      for (auto& c : unpublished_code) {
        state.publish_queue_.emplace_back(std::move(c));
      }
      return;
    }
    state.publisher_running_ = true;
  }
  while (true) {
    PublishCompilationResults(std::move(unpublished_code));
    unpublished_code.clear();

    // Keep publishing new code that came in.
    base::MutexGuard guard(&state.mutex_);
    DCHECK(state.publisher_running_);
    if (state.publish_queue_.empty()) {
      state.publisher_running_ = false;
      return;
    }
    unpublished_code.swap(state.publish_queue_);
  }
}

size_t CompilationStateImpl::NumOutstandingCompilations(
    CompilationTier tier) const {
  return compilation_unit_queues_.GetSizeForTier(tier);
}

void CompilationStateImpl::SetError() {
  compile_cancelled_.store(true, std::memory_order_relaxed);
  if (compile_failed_.exchange(true, std::memory_order_relaxed)) {
    return;  // Already failed before.
  }

  base::MutexGuard callbacks_guard(&callbacks_mutex_);
  TriggerOutstandingCallbacks();
  callbacks_.clear();
}

void CompilationStateImpl::WaitForBaselineCompileJob() {
  if (baseline_compile_job_->IsValid()) baseline_compile_job_->Join();
#ifdef DEBUG
  base::EnumSet<CompilationEvent> events{
      CompilationEvent::kFinishedBaselineCompilation,
      CompilationEvent::kFailedCompilation};
  base::MutexGuard guard(&callbacks_mutex_);
  DCHECK(finished_events_.contains_any(events));
#endif
}

void CompilationStateImpl::TierUpAllFunctions() {
  const WasmModule* module = native_module_->module();
  uint32_t num_wasm_functions = module->num_declared_functions;
  WasmCodeRefScope code_ref_scope;
  CompilationUnitBuilder builder(native_module_);
  for (uint32_t i = 0; i < num_wasm_functions; ++i) {
    int func_index = module->num_imported_functions + i;
    WasmCode* code = native_module_->GetCode(func_index);
    if (!code || !code->is_turbofan()) {
      builder.AddTopTierUnit(func_index, ExecutionTier::kTurbofan);
    }
  }
  builder.Commit();

  // Join the compilation, until no compilation units are left anymore.
  class DummyDelegate final : public JobDelegate {
    bool ShouldYield() override { return false; }
    bool IsJoiningThread() const override { return true; }
    void NotifyConcurrencyIncrease() override { UNIMPLEMENTED(); }
    uint8_t GetTaskId() override { return kMainTaskId; }
  };

  DummyDelegate delegate;
  ExecuteCompilationUnits(native_module_weak_, &delegate,
                          CompilationTier::kTopTier);

  // We cannot wait for other compilation threads to finish, so we explicitly
  // compile all functions which are not yet available as TurboFan code.
  for (uint32_t i = 0; i < num_wasm_functions; ++i) {
    uint32_t func_index = module->num_imported_functions + i;
    WasmCode* code = native_module_->GetCode(func_index);
    if (!code || !code->is_turbofan()) {
      wasm::GetWasmEngine()->CompileFunction(native_module_, func_index,
                                             wasm::ExecutionTier::kTurbofan);
    }
  }
}

std::shared_ptr<wasm::WasmWrapperHandle> CompileImportWrapperForTest(
    Isolate* isolate, ImportCallKind kind, const CanonicalSig* sig,
    int expected_arity, Suspend suspend) {
  if (v8_flags.wasm_jitless) {
    return nullptr;
  }

  return GetWasmImportWrapperCache()->GetCompiled(
      isolate, {kind, sig, expected_arity, suspend});
}

}  // namespace v8::internal::wasm

#undef TRACE_COMPILE
#undef TRACE_STREAMING
#undef TRACE_LAZY