deps/v8/src/maglev/maglev-assembler.h - main - githubmirror/node

deps/v8/src/maglev/maglev-assembler.h
1 203 строки52 KB
Michaël Zasso
deps: update V8 to 14.6.202.33
24 апр 2026, 19:01
24 апр 2026, 19:01f1e0b83
// Copyright 2022 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef V8_MAGLEV_MAGLEV_ASSEMBLER_H_
#define V8_MAGLEV_MAGLEV_ASSEMBLER_H_

#include "src/codegen/machine-type.h"
#include "src/codegen/macro-assembler.h"
#include "src/common/globals.h"
#include "src/flags/flags.h"
#include "src/interpreter/bytecode-flags-and-tokens.h"
#include "src/maglev/maglev-code-gen-state.h"
#include "src/maglev/maglev-ir.h"

namespace v8 {
namespace internal {
namespace maglev {

class Graph;
class MaglevAssembler;

inline IsolateFieldId SpaceAllocationTopAddress(AllocationType alloc_type) {
  if (alloc_type == AllocationType::kYoung) {
    return IsolateFieldId::kNewAllocationInfoTop;
  }
  DCHECK_EQ(alloc_type, AllocationType::kOld);
  return IsolateFieldId::kOldAllocationInfoTop;
}

inline IsolateFieldId SpaceAllocationLimitAddress(AllocationType alloc_type) {
  if (alloc_type == AllocationType::kYoung) {
    return IsolateFieldId::kNewAllocationInfoLimit;
  }
  DCHECK_EQ(alloc_type, AllocationType::kOld);
  return IsolateFieldId::kOldAllocationInfoLimit;
}

inline Builtin AllocateBuiltin(AllocationType alloc_type) {
  if (alloc_type == AllocationType::kYoung) {
    return Builtin::kAllocateInYoungGeneration;
  }
  DCHECK_EQ(alloc_type, AllocationType::kOld);
  return Builtin::kAllocateInOldGeneration;
}

// Label allowed to be passed to deferred code.
class ZoneLabelRef {
 public:
  explicit ZoneLabelRef(Zone* zone) : label_(zone->New<Label>()) {}
  explicit inline ZoneLabelRef(MaglevAssembler* masm);

  static ZoneLabelRef UnsafeFromLabelPointer(Label* label) {
    // This is an unsafe operation, {label} must be zone allocated.
    return ZoneLabelRef(label);
  }

  Label* operator*() { return label_; }

 private:
  Label* label_;

  // Unsafe constructor. {label} must be zone allocated.
  explicit ZoneLabelRef(Label* label) : label_(label) {}
};

// The slot index is the offset from the frame pointer.
struct StackSlot {
  int32_t index;
};

// Helper for generating the platform-specific parts of map comparison
// operations.
class MapCompare {
 public:
  inline explicit MapCompare(MaglevAssembler* masm, Register object,
                             size_t map_count);

  inline void Generate(Handle<Map> map, Condition cond, Label* if_true,
                       Label::Distance distance = Label::kFar);
  inline Register GetObject() const { return object_; }
  inline Register GetMap();

  // For counting the temporaries needed by the above operations:
  static inline int TemporaryCount(size_t map_count);

 private:
  MaglevAssembler* masm_;
  const Register object_;
  const size_t map_count_;
  Register map_ = Register::no_reg();
};

class V8_EXPORT_PRIVATE MaglevAssembler : public MacroAssembler {
 public:
  class TemporaryRegisterScope;

  MaglevAssembler(Isolate* isolate, Zone* zone,
                  MaglevCodeGenState* code_gen_state)
      : MacroAssembler(isolate, zone, CodeObjectRequired::kNo),
        code_gen_state_(code_gen_state) {}

  static constexpr RegList GetAllocatableRegisters() {
#if defined(V8_TARGET_ARCH_ARM)
    return kAllocatableGeneralRegisters - kMaglevExtraScratchRegister;
#elif defined(V8_TARGET_ARCH_RISCV64)
    return kAllocatableGeneralRegisters - kMaglevExtraScratchRegister -
           kMaglevFlagsRegister;
#elif defined(V8_TARGET_ARCH_LOONG64)
    return kAllocatableGeneralRegisters - kMaglevFlagsRegister;
#else
    return kAllocatableGeneralRegisters;
#endif
  }

#if defined(V8_TARGET_ARCH_RISCV64) || defined(V8_TARGET_ARCH_LOONG64)
  static constexpr Register GetFlagsRegister() { return kMaglevFlagsRegister; }
#endif  // V8_TARGET_ARCH_RISCV64 || V8_TARGET_ARCH_LOONG64

  static constexpr DoubleRegList GetAllocatableDoubleRegisters() {
    return kAllocatableDoubleRegisters;
  }

  inline MemOperand GetStackSlot(const compiler::AllocatedOperand& operand);
  inline MemOperand ToMemOperand(const compiler::InstructionOperand& operand);

  inline Register GetFramePointer();

  inline int GetFramePointerOffsetForStackSlot(
      const compiler::AllocatedOperand& operand) {
    int index = operand.index();
    if (operand.representation() != MachineRepresentation::kTagged) {
      index += code_gen_state()->tagged_slots();
    }
    return GetFramePointerOffsetForStackSlot(index);
  }

  template <typename Dest, typename Source>
  inline void MoveRepr(MachineRepresentation repr, Dest dst, Source src);

  void Allocate(RegisterSnapshot register_snapshot, Register result,
                int size_in_bytes,
                AllocationType alloc_type = AllocationType::kYoung,
                AllocationAlignment alignment = kTaggedAligned);

  void Allocate(RegisterSnapshot register_snapshot, Register result,
                Register size_in_bytes,
                AllocationType alloc_type = AllocationType::kYoung,
                AllocationAlignment alignment = kTaggedAligned);

  void AllocateHeapNumber(RegisterSnapshot register_snapshot, Register result,
                          DoubleRegister value);

  void AllocateTwoByteString(RegisterSnapshot register_snapshot,
                             Register result, int length);

  void LoadSingleCharacterString(Register result, int char_code);
  void LoadSingleCharacterString(Register result, Register char_code,
                                 Register scratch);

  void EnsureWritableFastElements(RegisterSnapshot register_snapshot,
                                  Register elements, Register object,
                                  Register scratch);

  inline void BindJumpTarget(Label* label);
  inline void BindBlock(BasicBlock* block);

  inline Condition IsRootConstant(Input input, RootIndex root_index);

  inline void Branch(Condition condition, BasicBlock* if_true,
                     BasicBlock* if_false, BasicBlock* next_block);
  inline void Branch(Condition condition, Label* if_true,
                     Label::Distance true_distance, bool fallthrough_when_true,
                     Label* if_false, Label::Distance false_distance,
                     bool fallthrough_when_false);
#if defined(V8_TARGET_ARCH_LOONG64)
  void Branch(Register r1, const Operand& r2, Condition condition,
              Label* if_true, bool fallthrough_when_true, Label* if_false,
              bool fallthrough_when_false);
#endif

  Register FromAnyToRegister(ConstInput input, Register scratch);

  inline void LoadTaggedField(Register result, MemOperand operand);
  inline void LoadTaggedField(Register result, Register object, int offset);
  inline void LoadTaggedFieldWithoutDecompressing(Register result,
                                                  Register object, int offset);
  inline void LoadTaggedSignedField(Register result, MemOperand operand);
  inline void LoadTaggedSignedField(Register result, Register object,
                                    int offset);
  inline void LoadAndUntagTaggedSignedField(Register result, Register object,
                                            int offset);
  inline void LoadTaggedFieldByIndex(Register result, Register object,
                                     Register index, int scale, int offset);
  inline void LoadBoundedSizeFromObject(Register result, Register object,
                                        int offset);
  inline void LoadExternalPointerField(Register result, MemOperand operand);

  inline void LoadFixedArrayElement(Register result, Register array,
                                    Register index);
  inline void LoadFixedArrayElementWithoutDecompressing(Register result,
                                                        Register array,
                                                        Register index);
  inline void LoadFixedDoubleArrayElement(DoubleRegister result, Register array,
                                          Register index);
  inline void StoreFixedDoubleArrayElement(Register array, Register index,
                                           DoubleRegister value);

  inline void LoadSignedField(Register result, MemOperand operand,
                              int element_size);
  inline void LoadUnsignedField(Register result, MemOperand operand,
                                int element_size);
  template <typename BitField>
  inline void LoadBitField(Register result, MemOperand operand) {
    static constexpr int load_size = sizeof(typename BitField::BaseType);
    LoadUnsignedField(result, operand, load_size);
    DecodeField<BitField>(result);
  }

  enum StoreMode { kField, kElement };
  enum ValueIsCompressed { kValueIsDecompressed, kValueIsCompressed };
  enum ValueCanBeSmi { kValueCannotBeSmi, kValueCanBeSmi };

  inline void SetSlotAddressForTaggedField(Register slot_reg, Register object,
                                           int offset);
  inline void SetSlotAddressForFixedArrayElement(Register slot_reg,
                                                 Register object,
                                                 Register index);

  template <StoreMode store_mode>
  using OffsetTypeFor = std::conditional_t<store_mode == kField, int, Register>;

  template <StoreMode store_mode>
  void CheckAndEmitDeferredWriteBarrier(Register object,
                                        OffsetTypeFor<store_mode> offset,
                                        Register value,
                                        RegisterSnapshot register_snapshot,
                                        ValueIsCompressed value_is_compressed,
                                        ValueCanBeSmi value_can_be_smi);

  void CheckAndEmitDeferredIndirectPointerWriteBarrier(
      Register object, int offset, Register value,
      RegisterSnapshot register_snapshot, IndirectPointerTag tag);

  // Preserves all registers that are in the register snapshot, but is otherwise
  // allowed to clobber both input registers if they are not in the snapshot.
  //
  // For maximum efficiency, prefer:
  //   * Having `object` == WriteBarrierDescriptor::ObjectRegister(),
  //   * Not having WriteBarrierDescriptor::SlotAddressRegister() in the
  //     register snapshot,
  //   * Not having `value` in the register snapshot, allowing it to be
  //     clobbered.
  void StoreTaggedFieldWithWriteBarrier(Register object, int offset,
                                        Register value,
                                        RegisterSnapshot register_snapshot,
                                        ValueIsCompressed value_is_compressed,
                                        ValueCanBeSmi value_can_be_smi);
  inline void StoreTaggedFieldNoWriteBarrier(Register object, int offset,
                                             Register value);
  inline void StoreTaggedSignedField(Register object, int offset,
                                     Register value);
  inline void StoreTaggedSignedField(Register object, int offset,
                                     Tagged<Smi> value);

  inline void StoreInt32Field(Register object, int offset, int32_t value);

  inline void AssertElidedWriteBarrier(Register object, Register value,
                                       RegisterSnapshot snapshot);

#ifdef V8_ENABLE_SANDBOX

  void StoreTrustedPointerFieldWithWriteBarrier(
      Register object, int offset, Register value,
      RegisterSnapshot register_snapshot, IndirectPointerTag tag);
  inline void StoreTrustedPointerFieldNoWriteBarrier(Register object,
                                                     int offset,
                                                     Register value);
#endif  // V8_ENABLE_SANDBOX

  inline void StoreField(MemOperand operand, Register value, int element_size);
  inline void ReverseByteOrder(Register value, int element_size);

  inline void BuildTypedArrayDataPointer(Register data_pointer,
                                         Register object);
  inline MemOperand TypedArrayElementOperand(Register data_pointer,
                                             Register index, int element_size);

  inline void StoreDataViewElement(Register value, Register data_pointer,
                                   Register index, int element_size);
  inline void LoadDataViewElement(Register result, Register data_pointer,
                                  Register index, int element_size);

  enum class CharCodeMaskMode { kValueIsInRange, kMustApplyMask };

  // Warning: Input registers {string} and {index} will be scratched.
  // {result} is allowed to alias with one the other 3 input registers.
  // {result} is an int32.
  void StringCharCodeOrCodePointAt(
      BuiltinStringPrototypeCharCodeOrCodePointAt::Mode mode,
      RegisterSnapshot& register_snapshot, Register result, Register string,
      Register index, Register scratch1, Register scratch2,
      Label* result_fits_one_byte);
  void SeqOneByteStringCharCodeAt(Register result, Register string,
                                  Register index);
  // Warning: Input {char_code} will be scratched.
  void StringFromCharCode(RegisterSnapshot register_snapshot,
                          Label* char_code_fits_one_byte, Register result,
                          Register char_code, Register scratch,
                          CharCodeMaskMode mask_mode);

  void ToBoolean(Register value, CheckType check_type, ZoneLabelRef is_true,
                 ZoneLabelRef is_false, bool fallthrough_when_true);

  void TestTypeOf(Register object,
                  interpreter::TestTypeOfFlags::LiteralFlag literal,
                  Label* if_true, Label::Distance true_distance,
                  bool fallthrough_when_true, Label* if_false,
                  Label::Distance false_distance, bool fallthrough_when_false);

  inline void SmiTagInt32AndJumpIfFail(Register dst, Register src, Label* fail,
                                       Label::Distance distance = Label::kFar);
  inline void SmiTagInt32AndJumpIfFail(Register reg, Label* fail,
                                       Label::Distance distance = Label::kFar);
  inline void SmiTagInt32AndJumpIfSuccess(
      Register dst, Register src, Label* success,
      Label::Distance distance = Label::kFar);
  inline void SmiTagInt32AndJumpIfSuccess(
      Register reg, Label* success, Label::Distance distance = Label::kFar);
  inline void UncheckedSmiTagInt32(Register dst, Register src);
  inline void UncheckedSmiTagInt32(Register reg);

  inline void SmiTagUint32AndJumpIfFail(Register dst, Register src, Label* fail,
                                        Label::Distance distance = Label::kFar);
  inline void SmiTagUint32AndJumpIfFail(Register reg, Label* fail,
                                        Label::Distance distance = Label::kFar);
  inline void SmiTagIntPtrAndJumpIfFail(Register dst, Register src, Label* fail,
                                        Label::Distance distance = Label::kFar);
  inline void SmiTagUint32AndJumpIfSuccess(
      Register dst, Register src, Label* success,
      Label::Distance distance = Label::kFar);
  inline void SmiTagUint32AndJumpIfSuccess(
      Register reg, Label* success, Label::Distance distance = Label::kFar);
  inline void SmiTagIntPtrAndJumpIfSuccess(
      Register dst, Register src, Label* success,
      Label::Distance distance = Label::kFar);
  inline void UncheckedSmiTagUint32(Register dst, Register src);
  inline void UncheckedSmiTagUint32(Register reg);

  // Try to smi-tag {obj}. Result is thrown away. Only use this when
  // SmiValuesAre31Bits().
  inline void CheckInt32IsSmi(Register obj, Label* fail,
                              Register scratch = Register::no_reg());

  inline void CheckIntPtrIsSmi(Register obj, Label* fail,
                               Label::Distance distance = Label::kFar);

  // Add/Subtract a constant (not smi tagged) to a smi. Jump to {fail} if the
  // result doesn't fit.
  inline void SmiAddConstant(Register dst, Register src, int value, Label* fail,
                             Label::Distance distance = Label::kFar);
  inline void SmiAddConstant(Register reg, int value, Label* fail,
                             Label::Distance distance = Label::kFar);
  inline void SmiSubConstant(Register dst, Register src, int value, Label* fail,
                             Label::Distance distance = Label::kFar);
  inline void SmiSubConstant(Register reg, int value, Label* fail,
                             Label::Distance distance = Label::kFar);

  inline void MoveHeapNumber(Register dst, double value);

#if defined(V8_TARGET_ARCH_RISCV64)
  inline Condition CheckSmi(Register src);
  // Abort execution if argument is not a Map, enabled via
  // --debug-code.
  void AssertMap(Register object) NOOP_UNLESS_DEBUG_CODE;

  void CompareRoot(const Register& obj, RootIndex index,
                   ComparisonMode mode = ComparisonMode::kDefault);
  void CmpTagged(const Register& rs1, const Register& rs2);
  void CompareTaggedRoot(const Register& obj, RootIndex index);
  void Cmp(const Register& rn, int imm);
  void Assert(Condition cond, AbortReason reason);
  void IsObjectType(Register heap_object, Register scratch1, Register scratch2,
                    InstanceType type);
#elif defined(V8_TARGET_ARCH_LOONG64)
  inline Condition CheckSmi(Register src);
  void Assert(Condition cond, AbortReason reason);

  void Cmp(const Register& rj, const Operand& operand);
  void Cmp(const Register& rj, const uint32_t imm);
  void CmpTagged(const Register& rj, const Operand& operand);
  void CompareRoot(const Register& obj, RootIndex index,
                   ComparisonMode mode = ComparisonMode::kDefault);
  void CompareTaggedRoot(const Register& obj, RootIndex index);
#endif

  void TruncateDoubleToInt32(Register dst, DoubleRegister src);
  void TryTruncateDoubleToInt32(Register dst, DoubleRegister src, Label* fail);
  void TryTruncateDoubleToUint32(Register dst, DoubleRegister src, Label* fail);

  void TryChangeFloat64ToIndex(Register result, DoubleRegister value,
                               Label* success, Label* fail);

  inline void MaybeEmitPlaceHolderForDeopt();
  inline void DefineLazyDeoptPoint(LazyDeoptInfo* info);
  inline void DefineExceptionHandlerPoint(NodeBase* node);
  inline void DefineExceptionHandlerAndLazyDeoptPoint(NodeBase* node);

  template <typename Function, typename... Args>
  inline Label* MakeDeferredCode(Function&& deferred_code_gen, Args&&... args);
  template <typename Function, typename... Args>
  inline void JumpToDeferredIf(Condition cond, Function&& deferred_code_gen,
                               Args&&... args);
  void JumpIfNotCallable(Register object, Register scratch,
                         CheckType check_type, Label* target,
                         Label::Distance distance = Label::kFar);
  void JumpIfUndetectable(Register object, Register scratch,
                          CheckType check_type, Label* target,
                          Label::Distance distance = Label::kFar);
  void JumpIfNotUndetectable(Register object, Register scratch, CheckType,
                             Label* target,
                             Label::Distance distance = Label::kFar);
  template <typename NodeT>
  inline Label* GetDeoptLabel(NodeT* node, DeoptimizeReason reason);
  inline bool IsDeoptLabel(Label* label);
  inline void EmitEagerDeoptStress(Label* label);
#ifdef V8_DUMPLING
  inline bool IsTopFrameInterpreted(Label* label);
#endif  // V8_DUMPLING
  template <typename NodeT>
  inline void EmitEagerDeopt(NodeT* node, DeoptimizeReason reason);
  template <typename NodeT>
  inline void EmitEagerDeoptIf(Condition cond, DeoptimizeReason reason,
                               NodeT* node);
  template <typename NodeT>
  inline void EmitEagerDeoptIfNotEqual(DeoptimizeReason reason, NodeT* node);
  template <typename NodeT>
  inline void EmitEagerDeoptIfSmi(NodeT* node, Register object,
                                  DeoptimizeReason reason);
  template <typename NodeT>
  inline void EmitEagerDeoptIfNotSmi(NodeT* node, Register object,
                                     DeoptimizeReason reason);

  void MaterialiseValueNode(Register dst, ValueNode* value);

  inline void IncrementInt32(Register reg);
  inline void DecrementInt32(Register reg);
  inline void AddInt32(Register reg, int amount);
  inline void AddInt32(Register reg, Register other);
  inline void AndInt32(Register reg, int mask);
  inline void OrInt32(Register reg, int mask);
  inline void AndInt32(Register reg, Register other);
  inline void OrInt32(Register reg, Register other);
  inline void ShiftLeft(Register reg, int amount);
  inline void IncrementAddress(Register reg, int32_t delta);
  inline void LoadAddress(Register dst, MemOperand location);

  inline void EmitEnterExitFrame(int extra_slots, StackFrame::Type frame_type,
                                 Register scratch);

  inline MemOperand StackSlotOperand(StackSlot slot);
  inline void Move(StackSlot dst, Register src);
  inline void Move(StackSlot dst, DoubleRegister src);
  inline void Move(Register dst, StackSlot src);
  inline void Move(DoubleRegister dst, StackSlot src);
  inline void Move(MemOperand dst, Register src);
  inline void Move(Register dst, MemOperand src);
  inline void Move(DoubleRegister dst, DoubleRegister src);
  inline void Move(Register dst, Tagged<Smi> src);
  inline void Move(Register dst, ExternalReference src);
  inline void Move(Register dst, Register src);
  inline void Move(Register dst, Tagged<TaggedIndex> i);
  inline void Move(Register dst, int32_t i);
  inline void Move(Register dst, uint32_t i);
#ifdef V8_TARGET_ARCH_64_BIT
  inline void Move(Register dst, intptr_t p);
#endif
  inline void Move(Register dst, IndirectPointerTag i);
  inline void Move(DoubleRegister dst, double n);
  inline void Move(DoubleRegister dst, Float64 n);
  inline void Move(Register dst, Handle<HeapObject> obj);

  void Move(ExternalReference dst, int32_t imm);

  inline void MoveTagged(Register dst, Handle<HeapObject> obj);

  inline void LoadMapForCompare(Register dst, Register obj);

  inline void LoadByte(Register dst, MemOperand src);

  inline void LoadInt32(Register dst, MemOperand src);
  inline void StoreInt32(MemOperand dst, Register src);

  inline void LoadFloat32(DoubleRegister dst, MemOperand src);
  inline void StoreFloat32(MemOperand dst, DoubleRegister src);
  inline void LoadFloat64(DoubleRegister dst, MemOperand src);
  inline void StoreFloat64(MemOperand dst, DoubleRegister src);

  inline void LoadUnalignedFloat64(DoubleRegister dst, Register base,
                                   Register index);
  inline void LoadUnalignedFloat64AndReverseByteOrder(DoubleRegister dst,
                                                      Register base,
                                                      Register index);
  inline void StoreUnalignedFloat64(Register base, Register index,
                                    DoubleRegister src);
  inline void ReverseByteOrderAndStoreUnalignedFloat64(Register base,
                                                       Register index,
                                                       DoubleRegister src);

  inline void SignExtend32To64Bits(Register dst, Register src);
  inline void NegateInt32(Register val);

  void CountLeadingZerosInt32(Register dst, Register src);

  inline void ToUint8Clamped(Register result, DoubleRegister value, Label* min,
                             Label* max, Label* done);

  template <typename NodeT>
  inline void DeoptIfBufferNotValid(Register array, Register scratch,
                                    TypedArrayAccessMode mode, NodeT* node);

  inline Condition IsCallableAndNotUndetectable(Register map, Register scratch);
  inline Condition IsNotCallableNorUndetactable(Register map, Register scratch);

  inline void LoadInstanceType(Register instance_type, Register heap_object);
  inline void JumpIfObjectType(Register heap_object, InstanceType type,
                               Label* target,
                               Label::Distance distance = Label::kFar);
  inline void JumpIfNotObjectType(Register heap_object, InstanceType type,
                                  Label* target,
                                  Label::Distance distance = Label::kFar);
  inline void AssertObjectType(Register heap_object, InstanceType type,
                               AbortReason reason);
  inline void BranchOnObjectType(Register heap_object, InstanceType type,
                                 Label* if_true, Label::Distance true_distance,
                                 bool fallthrough_when_true, Label* if_false,
                                 Label::Distance false_distance,
                                 bool fallthrough_when_false);

  inline void JumpIfObjectTypeInRange(Register heap_object,
                                      InstanceType lower_limit,
                                      InstanceType higher_limit, Label* target,
                                      Label::Distance distance = Label::kFar);
  inline void JumpIfObjectTypeNotInRange(
      Register heap_object, InstanceType lower_limit, InstanceType higher_limit,
      Label* target, Label::Distance distance = Label::kFar);
  inline void AssertObjectTypeInRange(Register heap_object,
                                      InstanceType lower_limit,
                                      InstanceType higher_limit,
                                      AbortReason reason);
  inline void BranchOnObjectTypeInRange(
      Register heap_object, InstanceType lower_limit, InstanceType higher_limit,
      Label* if_true, Label::Distance true_distance, bool fallthrough_when_true,
      Label* if_false, Label::Distance false_distance,
      bool fallthrough_when_false);

#if V8_STATIC_ROOTS_BOOL
  inline void JumpIfObjectInRange(Register heap_object, Tagged_t lower_limit,
                                  Tagged_t higher_limit, Label* target,
                                  Label::Distance distance = Label::kFar);
  inline void JumpIfObjectNotInRange(Register heap_object, Tagged_t lower_limit,
                                     Tagged_t higher_limit, Label* target,
                                     Label::Distance distance = Label::kFar);
  inline void AssertObjectInRange(Register heap_object, Tagged_t lower_limit,
                                  Tagged_t higher_limit, AbortReason reason);
#endif

  inline void JumpIfJSAnyIsNotPrimitive(Register heap_object, Label* target,
                                        Label::Distance distance = Label::kFar);

  inline void JumpIfStringMap(Register map, Label* target,
                              Label::Distance distance = Label::kFar,
                              bool jump_if_true = true);
  inline void JumpIfSeqOneByteStringMap(Register map, Label* target,
                                        Label::Distance distance = Label::kFar,
                                        bool jump_if_true = true);
  inline void JumpIfString(Register heap_object, Label* target,
                           Label::Distance distance = Label::kFar);
  inline void JumpIfNotString(Register heap_object, Label* target,
                              Label::Distance distance = Label::kFar);
  inline void JumpIfNotSeqOneByteString(Register heap_object, Label* target,
                                        Label::Distance distance = Label::kFar);
  inline void CheckJSAnyIsStringAndBranch(Register heap_object, Label* if_true,
                                          Label::Distance true_distance,
                                          bool fallthrough_when_true,
                                          Label* if_false,
                                          Label::Distance false_distance,
                                          bool fallthrough_when_false);

  inline void CompareMapWithRoot(Register object, RootIndex index,
                                 Register scratch);

  inline void CompareInstanceTypeAndJumpIf(Register map, InstanceType type,
                                           Condition cond, Label* target,
                                           Label::Distance distance);

  inline void CompareInstanceType(Register map, InstanceType instance_type);
  inline void CompareInstanceTypeRange(Register map, InstanceType lower_limit,
                                       InstanceType higher_limit);
  inline Condition CompareInstanceTypeRange(Register map,
                                            Register instance_type_out,
                                            InstanceType lower_limit,
                                            InstanceType higher_limit);

  template <typename NodeT>
  inline void CompareInstanceTypeRangeAndEagerDeoptIf(
      Register map, Register instance_type_out, InstanceType lower_limit,
      InstanceType higher_limit, Condition cond, DeoptimizeReason reason,
      NodeT* node);

  template <typename NodeT>
  void CompareRootAndEmitEagerDeoptIf(Register reg, RootIndex index,
                                      Condition cond, DeoptimizeReason reason,
                                      NodeT* node);
  template <typename NodeT>
  void CompareMapWithRootAndEmitEagerDeoptIf(Register reg, RootIndex index,
                                             Register scratch, Condition cond,
                                             DeoptimizeReason reason,
                                             NodeT* node);
  template <typename NodeT>
  void CompareTaggedRootAndEmitEagerDeoptIf(Register reg, RootIndex index,
                                            Condition cond,
                                            DeoptimizeReason reason,
                                            NodeT* node);
  template <typename NodeT>
  void CompareUInt32AndEmitEagerDeoptIf(Register reg, int imm, Condition cond,
                                        DeoptimizeReason reason, NodeT* node);
  inline void CompareTaggedAndJumpIf(Register reg, Tagged<Smi> smi,
                                     Condition cond, Label* target,
                                     Label::Distance distance = Label::kFar);
  inline void CompareTaggedAndJumpIf(Register reg, Handle<HeapObject> obj,
                                     Condition cond, Label* target,
                                     Label::Distance distance = Label::kFar);
  inline void CompareTaggedAndJumpIf(Register src1, Register src2,
                                     Condition cond, Label* target,
                                     Label::Distance distance = Label::kFar);

  inline void CompareFloat64AndJumpIf(DoubleRegister src1, DoubleRegister src2,
                                      Condition cond, Label* target,
                                      Label* nan_failed,
                                      Label::Distance distance = Label::kFar);
  inline void CompareFloat64AndBranch(DoubleRegister src1, DoubleRegister src2,
                                      Condition cond, BasicBlock* if_true,
                                      BasicBlock* if_false,
                                      BasicBlock* next_block,
                                      BasicBlock* nan_failed);
  inline void PrepareCallCFunction(int num_reg_arguments,
                                   int num_double_registers = 0);

  inline void CallSelf();
  inline void CallJSBuiltin(Builtin builtin, uint16_t parameter_count);
  inline void CallBuiltin(Builtin builtin);
  template <Builtin kBuiltin, typename... Args>
  inline void CallBuiltin(Args&&... args);
  inline void CallRuntime(Runtime::FunctionId fid);
  inline void CallRuntime(Runtime::FunctionId fid, int num_args);

  inline void Jump(Label* target, Label::Distance distance = Label::kFar);
  inline void JumpToDeopt(Label* target);
  inline void JumpIf(Condition cond, Label* target,
                     Label::Distance distance = Label::kFar);

  inline void JumpIfRoot(Register with, RootIndex index, Label* if_equal,
                         Label::Distance distance = Label::kFar);
  inline void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal,
                            Label::Distance distance = Label::kFar);
  inline void JumpIfSmi(Register src, Label* on_smi,
                        Label::Distance near_jump = Label::kFar);
  inline void JumpIfNotSmi(Register src, Label* on_not_smi,
                           Label::Distance near_jump = Label::kFar);
  inline void JumpIfByte(Condition cc, Register value, int32_t byte,
                         Label* target, Label::Distance distance = Label::kFar);

  inline void Float64SilenceNan(DoubleRegister value);
#ifdef V8_ENABLE_UNDEFINED_DOUBLE
  inline void JumpIfUndefinedNan(DoubleRegister value, Register scratch,
                                 Label* target,
                                 Label::Distance distance = Label::kFar);
  inline void JumpIfNotUndefinedNan(DoubleRegister value, Register scratch,
                                    Label* target,
                                    Label::Distance distance = Label::kFar);
  inline void JumpIfUndefinedNan(MemOperand operand, Label* target,
                                 Label::Distance distance = Label::kFar);
#endif  // V8_ENABLE_UNDEFINED_DOUBLE
  inline void JumpIfHoleNan(DoubleRegister value, Register scratch,
                            Label* target,
                            Label::Distance distance = Label::kFar);
  inline void JumpIfNotHoleNan(DoubleRegister value, Register scratch,
                               Label* target,
                               Label::Distance distance = Label::kFar);
  inline void JumpIfNan(DoubleRegister value, Label* target,
                        Label::Distance distance = Label::kFar);
  inline void JumpIfNotNan(DoubleRegister value, Label* target,
                           Label::Distance distance = Label::kFar);
  inline void JumpIfHoleNan(MemOperand operand, Label* target,
                            Label::Distance distance = Label::kFar);
  inline void JumpIfNotHoleNan(MemOperand operand, Label* target,
                               Label::Distance distance = Label::kFar);

  inline void CompareInt32AndJumpIf(Register r1, Register r2, Condition cond,
                                    Label* target,
                                    Label::Distance distance = Label::kFar);
  inline void CompareIntPtrAndJumpIf(Register r1, Register r2, Condition cond,
                                     Label* target,
                                     Label::Distance distance = Label::kFar);
  inline void CompareIntPtrAndJumpIf(Register r1, int32_t value, Condition cond,
                                     Label* target,
                                     Label::Distance distance = Label::kFar);
  inline void CompareInt32AndJumpIf(Register r1, int32_t value, Condition cond,
                                    Label* target,
                                    Label::Distance distance = Label::kFar);
  inline void CompareInt32AndBranch(Register r1, int32_t value, Condition cond,
                                    BasicBlock* if_true, BasicBlock* if_false,
                                    BasicBlock* next_block);
  inline void CompareInt32AndBranch(Register r1, Register r2, Condition cond,
                                    BasicBlock* if_true, BasicBlock* if_false,
                                    BasicBlock* next_block);
  inline void CompareInt32AndBranch(Register r1, int32_t value, Condition cond,
                                    Label* if_true,
                                    Label::Distance true_distance,
                                    bool fallthrough_when_true, Label* if_false,
                                    Label::Distance false_distance,
                                    bool fallthrough_when_false);
  inline void CompareInt32AndBranch(Register r1, Register r2, Condition cond,
                                    Label* if_true,
                                    Label::Distance true_distance,
                                    bool fallthrough_when_true, Label* if_false,
                                    Label::Distance false_distance,
                                    bool fallthrough_when_false);
  inline void CompareIntPtrAndBranch(Register r1, int32_t value, Condition cond,
                                     BasicBlock* if_true, BasicBlock* if_false,
                                     BasicBlock* next_block);
  inline void CompareIntPtrAndBranch(Register r1, int32_t value, Condition cond,
                                     Label* if_true,
                                     Label::Distance true_distance,
                                     bool fallthrough_when_true,
                                     Label* if_false,
                                     Label::Distance false_distance,
                                     bool fallthrough_when_false);
  inline void CompareInt32AndAssert(Register r1, Register r2, Condition cond,
                                    AbortReason reason);
  inline void CompareInt32AndAssert(Register r1, int32_t value, Condition cond,
                                    AbortReason reason);
  inline void CompareSmiAndJumpIf(Register r1, Tagged<Smi> value,
                                  Condition cond, Label* target,
                                  Label::Distance distance = Label::kFar);
  inline void CompareSmiAndAssert(Register r1, Tagged<Smi> value,
                                  Condition cond, AbortReason reason);
  inline void CompareByteAndJumpIf(MemOperand left, int8_t right,
                                   Condition cond, Register scratch,
                                   Label* target,
                                   Label::Distance distance = Label::kFar);

  inline void CompareDoubleAndJumpIfZeroOrNaN(
      DoubleRegister reg, Label* target,
      Label::Distance distance = Label::kFar);
  inline void CompareDoubleAndJumpIfZeroOrNaN(
      MemOperand operand, Label* target,
      Label::Distance distance = Label::kFar);

  inline void TestInt32AndJumpIfAnySet(Register r1, int32_t mask, Label* target,
                                       Label::Distance distance = Label::kFar);
  inline void TestInt32AndJumpIfAnySet(MemOperand operand, int32_t mask,
                                       Label* target,
                                       Label::Distance distance = Label::kFar);
  inline void TestUint8AndJumpIfAnySet(MemOperand operand, uint8_t mask,
                                       Label* target,
                                       Label::Distance distance = Label::kFar);

  inline void TestInt32AndJumpIfAllClear(
      Register r1, int32_t mask, Label* target,
      Label::Distance distance = Label::kFar);
  inline void TestInt32AndJumpIfAllClear(
      MemOperand operand, int32_t mask, Label* target,
      Label::Distance distance = Label::kFar);
  inline void TestUint8AndJumpIfAllClear(
      MemOperand operand, uint8_t mask, Label* target,
      Label::Distance distance = Label::kFar);

  inline void Int32ToDouble(DoubleRegister result, Register src);
  inline void Uint32ToDouble(DoubleRegister result, Register src);
  inline void SmiToDouble(DoubleRegister result, Register smi);
  inline void IntPtrToDouble(DoubleRegister result, Register src);

  inline void StringLength(Register result, Register string);
  inline void LoadThinStringValue(Register result, Register string);

  // The registers WriteBarrierDescriptor::ObjectRegister and
  // WriteBarrierDescriptor::SlotAddressRegister can be clobbered.
  void StoreFixedArrayElementWithWriteBarrier(
      Register array, Register index, Register value,
      RegisterSnapshot register_snapshot);
  inline void StoreFixedArrayElementNoWriteBarrier(Register array,
                                                   Register index,
                                                   Register value);

  // TODO(victorgomes): Import baseline Pop(T...) methods.
  inline void Pop(Register dst);
  using MacroAssembler::Pop;

  template <typename... T>
  inline void Push(T... vals);
  template <typename... T>
  inline void PushReverse(T... vals);

  void OSRPrologue(Graph* graph);
  void Prologue(Graph* graph);

  inline void FinishCode();

  inline void AssertStackSizeCorrect();
  inline Condition FunctionEntryStackCheck(int stack_check_offset);

  inline void SetMapAsRoot(Register object, RootIndex map);

  inline void AssertContextCellState(Register cell, ContextCell::State state,
                                     Condition condition = kEqual);
  inline void LoadContextCellState(Register state, Register cell);

  inline void LoadContextCellTaggedValue(Register value, Register cell);
  inline void StoreContextCellSmiValue(Register cell, Register value);

  inline void LoadContextCellInt32Value(Register value, Register cell);
  inline void StoreContextCellInt32Value(Register cell, Register value);

  inline void LoadContextCellFloat64Value(DoubleRegister value, Register cell);
  inline void StoreContextCellFloat64Value(Register cell, DoubleRegister value);

  inline void LoadHeapNumberValue(DoubleRegister result, Register heap_number);
  inline void StoreHeapNumberValue(DoubleRegister value, Register heap_number);

  inline void LoadHeapNumberOrOddballValue(DoubleRegister result,
                                           Register object);

  void LoadDataField(const PolymorphicAccessInfo& access_info, Register result,
                     Register object, Register scratch);

  void MaybeEmitDeoptBuiltinsCall(size_t eager_deopt_count,
                                  Label* eager_deopt_entry,
                                  size_t lazy_deopt_count,
                                  Label* lazy_deopt_entry);

  void TryMigrateInstance(Register object, RegisterSnapshot& register_snapshot,
                          Label* fail);

  void TryMigrateInstanceAndMarkMapAsMigrationTarget(
      Register object, RegisterSnapshot& register_snapshot);

  void ResetLastYoungAllocation();

  compiler::NativeContextRef native_context() const {
    return code_gen_state()->broker()->target_native_context();
  }

  MaglevCodeGenState* code_gen_state() const { return code_gen_state_; }
  MaglevSafepointTableBuilder* safepoint_table_builder() const {
    return code_gen_state()->safepoint_table_builder();
  }
  MaglevCompilationInfo* compilation_info() const {
    return code_gen_state()->compilation_info();
  }

  TemporaryRegisterScope* scratch_register_scope() const {
    return scratch_register_scope_;
  }

#ifdef DEBUG
  bool allow_allocate() const { return allow_allocate_; }
  void set_allow_allocate(bool value) { allow_allocate_ = value; }

  bool allow_call() const { return allow_call_; }
  void set_allow_call(bool value) { allow_call_ = value; }

  bool allow_deferred_call() const { return allow_deferred_call_; }
  void set_allow_deferred_call(bool value) { allow_deferred_call_ = value; }
#endif  // DEBUG

 private:
  template <typename Derived>
  class TemporaryRegisterScopeBase;

  inline constexpr int GetFramePointerOffsetForStackSlot(int index) {
    return StandardFrameConstants::kExpressionsOffset -
           index * kSystemPointerSize;
  }

  inline void CallBuiltinImpl(Builtin builtin);

  // Returns the condition code satisfied if tagging is successful. Only use
  // this when SmiValuesAre31Bits(); otherwise use SmiTag(dst, src).
  inline Condition TrySmiTagInt32(Register dst, Register src);

  MaglevCodeGenState* const code_gen_state_;
  TemporaryRegisterScope* scratch_register_scope_ = nullptr;
#ifdef DEBUG
  bool allow_allocate_ = false;
  bool allow_call_ = false;
  bool allow_deferred_call_ = false;
#endif  // DEBUG
};

// Shared logic for per-architecture TemporaryRegisterScope.
template <typename Derived>
class MaglevAssembler::TemporaryRegisterScopeBase {
 public:
  struct SavedData {
    RegList available_;
    DoubleRegList available_double_;
  };

  explicit TemporaryRegisterScopeBase(MaglevAssembler* masm)
      : masm_(masm),
        prev_scope_(masm->scratch_register_scope_),
        available_(masm->scratch_register_scope_
                       ? static_cast<TemporaryRegisterScopeBase*>(prev_scope_)
                             ->available_
                       : RegList()),
        available_double_(
            masm->scratch_register_scope_
                ? static_cast<TemporaryRegisterScopeBase*>(prev_scope_)
                      ->available_double_
                : DoubleRegList()) {
    masm_->scratch_register_scope_ = static_cast<Derived*>(this);
  }
  explicit TemporaryRegisterScopeBase(MaglevAssembler* masm,
                                      const SavedData& saved_data)
      : masm_(masm),
        prev_scope_(masm->scratch_register_scope_),
        available_(saved_data.available_),
        available_double_(saved_data.available_double_) {
    masm_->scratch_register_scope_ = static_cast<Derived*>(this);
  }
  ~TemporaryRegisterScopeBase() {
    masm_->scratch_register_scope_ = prev_scope_;
    // TODO(leszeks): Clear used registers.
  }

  void ResetToDefault() {
    available_ = {};
    available_double_ = {};
    static_cast<Derived*>(this)->ResetToDefaultImpl();
  }

  Register Acquire() {
    CHECK(!available_.is_empty());
    return available_.PopFirst();
  }
  void Include(const RegList list) {
    DCHECK((list - kAllocatableGeneralRegisters).is_empty());
    available_ = available_ | list;
  }

  DoubleRegister AcquireDouble() {
    CHECK(!available_double_.is_empty());
    return available_double_.PopFirst();
  }
  void IncludeDouble(const DoubleRegList list) {
    DCHECK((list - kAllocatableDoubleRegisters).is_empty());
    available_double_ = available_double_ | list;
  }

  RegList Available() { return available_; }
  void SetAvailable(RegList list) { available_ = list; }

  DoubleRegList AvailableDouble() { return available_double_; }
  void SetAvailableDouble(DoubleRegList list) { available_double_ = list; }

 protected:
  SavedData CopyForDeferBase() {
    return SavedData{available_, available_double_};
  }

  MaglevAssembler* masm_;
  Derived* prev_scope_;
  RegList available_;
  DoubleRegList available_double_;
};

class SaveRegisterStateForCall {
 public:
  SaveRegisterStateForCall(MaglevAssembler* masm, RegisterSnapshot snapshot)
      : masm(masm), snapshot_(snapshot) {
    masm->PushAll(snapshot_.live_registers);
    masm->PushAll(snapshot_.live_double_registers, kDoubleSize);
  }

  ~SaveRegisterStateForCall() {
    masm->PopAll(snapshot_.live_double_registers, kDoubleSize);
    masm->PopAll(snapshot_.live_registers);
  }

  void DefineSafepoint() {
    // TODO(leszeks): Avoid emitting safepoints when there are no registers to
    // save.
    auto safepoint = masm->safepoint_table_builder()->DefineSafepoint(masm);
    int pushed_reg_index = 0;
    for (Register reg : snapshot_.live_registers) {
      if (snapshot_.live_tagged_registers.has(reg)) {
        safepoint.DefineTaggedRegister(pushed_reg_index);
      }
      pushed_reg_index++;
    }
#ifdef V8_TARGET_ARCH_ARM64
    pushed_reg_index = RoundUp<2>(pushed_reg_index);
#endif
    int num_double_slots = snapshot_.live_double_registers.Count() *
                           (kDoubleSize / kSystemPointerSize);
#ifdef V8_TARGET_ARCH_ARM64
    num_double_slots = RoundUp<2>(num_double_slots);
#endif
    safepoint.SetNumExtraSpillSlots(pushed_reg_index + num_double_slots);
  }

  inline void DefineSafepointWithLazyDeopt(LazyDeoptInfo* lazy_deopt_info);

 private:
  MaglevAssembler* masm;
  RegisterSnapshot snapshot_;
};

ZoneLabelRef::ZoneLabelRef(MaglevAssembler* masm)
    : ZoneLabelRef(masm->compilation_info()->zone()) {}

// ---
// Deopt
// ---

inline bool MaglevAssembler::IsDeoptLabel(Label* label) {
  for (auto deopt : code_gen_state_->eager_deopts()) {
    if (deopt->deopt_entry_label() == label) {
      return true;
    }
  }
  return false;
}

#ifdef V8_DUMPLING
inline bool MaglevAssembler::IsTopFrameInterpreted(Label* label) {
  DCHECK(IsDeoptLabel(label));
  for (auto deopt : code_gen_state_->eager_deopts()) {
    if (deopt->deopt_entry_label() == label) {
      return deopt->top_frame().type() ==
             DeoptFrame::FrameType::kInterpretedFrame;
    }
  }
  UNREACHABLE();
}
#endif  // V8_DUMPLING

template <typename NodeT>
inline Label* MaglevAssembler::GetDeoptLabel(NodeT* node,
                                             DeoptimizeReason reason) {
  static_assert(NodeT::kProperties.can_eager_deopt());
  EagerDeoptInfo* deopt_info = node->eager_deopt_info();
  if (deopt_info->reason() != DeoptimizeReason::kUnknown) {
    DCHECK_EQ(deopt_info->reason(), reason);
  }
  if (deopt_info->deopt_entry_label()->is_unused()) {
    code_gen_state()->PushEagerDeopt(deopt_info);
    deopt_info->set_reason(reason);
  }
  return node->eager_deopt_info()->deopt_entry_label();
}

template <typename NodeT>
inline void MaglevAssembler::EmitEagerDeopt(NodeT* node,
                                            DeoptimizeReason reason) {
  RecordComment("-- jump to eager deopt");
  JumpToDeopt(GetDeoptLabel(node, reason));
}

template <typename NodeT>
inline void MaglevAssembler::EmitEagerDeoptIf(Condition cond,
                                              DeoptimizeReason reason,
                                              NodeT* node) {
  RecordComment("-- Jump to eager deopt");
  JumpIf(cond, GetDeoptLabel(node, reason));
}

template <typename NodeT>
void MaglevAssembler::EmitEagerDeoptIfSmi(NodeT* node, Register object,
                                          DeoptimizeReason reason) {
  RecordComment("-- Jump to eager deopt");
  JumpIfSmi(object, GetDeoptLabel(node, reason));
}

template <typename NodeT>
void MaglevAssembler::EmitEagerDeoptIfNotSmi(NodeT* node, Register object,
                                             DeoptimizeReason reason) {
  RecordComment("-- Jump to eager deopt");
  JumpIfNotSmi(object, GetDeoptLabel(node, reason));
}


// Helpers for pushing arguments.
template <typename T>
class RepeatIterator {
 public:
  // Although we pretend to be a random access iterator, only methods that are
  // required for Push() are implemented right now.
  typedef std::random_access_iterator_tag iterator_category;
  typedef T value_type;
  typedef int difference_type;
  typedef T* pointer;
  typedef T reference;
  RepeatIterator(T val, int count) : val_(val), count_(count) {}
  reference operator*() const { return val_; }
  pointer operator->() { return &val_; }
  RepeatIterator& operator++() {
    ++count_;
    return *this;
  }
  RepeatIterator& operator--() {
    --count_;
    return *this;
  }
  RepeatIterator& operator+=(difference_type diff) {
    count_ += diff;
    return *this;
  }
  bool operator!=(const RepeatIterator<T>& that) const {
    return count_ != that.count_;
  }
  bool operator==(const RepeatIterator<T>& that) const {
    return count_ == that.count_;
  }
  difference_type operator-(const RepeatIterator<T>& it) const {
    return count_ - it.count_;
  }

 private:
  T val_;
  int count_;
};

template <typename T>
auto RepeatValue(T val, int count) {
  return base::make_iterator_range(RepeatIterator<T>(val, 0),
                                   RepeatIterator<T>(val, count));
}

namespace detail {

template <class T>
struct is_iterator_range : std::false_type {};
template <typename T>
struct is_iterator_range<base::iterator_range<T>> : std::true_type {};

}  // namespace detail

// General helpers.

inline Condition ToCondition(AssertCondition cond) {
  switch (cond) {
#define CASE(Name)               \
  case AssertCondition::k##Name: \
    return k##Name;
    ASSERT_CONDITION(CASE)
#undef CASE
  }
}

constexpr Condition ConditionFor(Operation operation) {
  switch (operation) {
    case Operation::kEqual:
    case Operation::kStrictEqual:
      return kEqual;
    case Operation::kLessThan:
      return kLessThan;
    case Operation::kLessThanOrEqual:
      return kLessThanEqual;
    case Operation::kGreaterThan:
      return kGreaterThan;
    case Operation::kGreaterThanOrEqual:
      return kGreaterThanEqual;
    default:
      UNREACHABLE();
  }
}

constexpr Condition UnsignedConditionFor(Operation operation) {
  switch (operation) {
    case Operation::kEqual:
    case Operation::kStrictEqual:
      return kEqual;
    case Operation::kLessThan:
      return kUnsignedLessThan;
    case Operation::kLessThanOrEqual:
      return kUnsignedLessThanEqual;
    case Operation::kGreaterThan:
      return kUnsignedGreaterThan;
    case Operation::kGreaterThanOrEqual:
      return kUnsignedGreaterThanEqual;
    default:
      UNREACHABLE();
  }
}

}  // namespace maglev
}  // namespace internal
}  // namespace v8

#endif  // V8_MAGLEV_MAGLEV_ASSEMBLER_H_