githubmirror/cpython

Python/jit_unwind.c
1 049 строк41 KB
Diego Russo
GH-126910: Add GNU backtrace support for unwinding JIT frames (#149104)
05 май 2026, 11:29
05 май 2026, 11:291e5d942
/*
 * Python JIT - DWARF .eh_frame builder
 *
 * This file contains the DWARF CFI generator used to build .eh_frame
 * data for JIT code (perf jitdump and other unwinders).
 */

#include "Python.h"
#include "pycore_jit_unwind.h"
#include "pycore_lock.h"

#if defined(PY_HAVE_JIT_GDB_UNWIND)
#  include "jit_unwind_info.h"
#  if !JIT_UNWIND_INFO_SUPPORTED
#    error "JIT unwind info was not generated for this target"
#  endif
#endif

#if defined(PY_HAVE_PERF_TRAMPOLINE) \
    || defined(PY_HAVE_JIT_GDB_UNWIND) \
    || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND)

#if defined(PY_HAVE_JIT_GDB_UNWIND)
#  include <elf.h>
#endif
#if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND)
/*
 * libgcc exposes frame registration entry points, but GCC's public headers
 * on some distributions do not declare them even though the symbols are
 * available in libgcc_s.
 */
void __register_frame(const void *);
void __deregister_frame(const void *);
#endif
#include <stdio.h>
#include <string.h>

// =============================================================================
//                              DWARF CONSTANTS
// =============================================================================

/*
 * DWARF (Debug With Arbitrary Record Formats) constants
 *
 * DWARF is a debugging data format used to provide stack unwinding information.
 * These constants define the various encoding types and opcodes used in
 * DWARF Call Frame Information (CFI) records.
 */

/* DWARF Call Frame Information version */
#define DWRF_CIE_VERSION 1

/* DWARF CFA (Call Frame Address) opcodes */
enum {
    DWRF_CFA_nop = 0x0,                    // No operation
    DWRF_CFA_offset_extended = 0x5,        // Extended offset instruction
    DWRF_CFA_def_cfa = 0xc,               // Define CFA rule
    DWRF_CFA_def_cfa_register = 0xd,      // Define CFA register
    DWRF_CFA_def_cfa_offset = 0xe,        // Define CFA offset
    DWRF_CFA_offset_extended_sf = 0x11,   // Extended signed offset
    DWRF_CFA_advance_loc = 0x40,          // Advance location counter
    DWRF_CFA_offset = 0x80,               // Simple offset instruction
    DWRF_CFA_restore = 0xc0               // Restore register
};

/*
 * Architecture-specific DWARF register numbers
 *
 * These constants define the register numbering scheme used by DWARF
 * for each supported architecture. The numbers must match the ABI
 * specification for proper stack unwinding.
 */
enum {
#ifdef __x86_64__
    /* x86_64 register numbering (note: order is defined by x86_64 ABI) */
    DWRF_REG_AX,    // RAX
    DWRF_REG_DX,    // RDX
    DWRF_REG_CX,    // RCX
    DWRF_REG_BX,    // RBX
    DWRF_REG_SI,    // RSI
    DWRF_REG_DI,    // RDI
    DWRF_REG_BP,    // RBP
    DWRF_REG_SP,    // RSP
    DWRF_REG_8,     // R8
    DWRF_REG_9,     // R9
    DWRF_REG_10,    // R10
    DWRF_REG_11,    // R11
    DWRF_REG_12,    // R12
    DWRF_REG_13,    // R13
    DWRF_REG_14,    // R14
    DWRF_REG_15,    // R15
    DWRF_REG_RA,    // Return address (RIP)
#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__)
    /* AArch64 register numbering */
    DWRF_REG_FP = 29,  // Frame Pointer
    DWRF_REG_RA = 30,  // Link register (return address)
    DWRF_REG_SP = 31,  // Stack pointer
#else
#    error "Unsupported target architecture"
#endif
};

// =============================================================================
//                              ELF OBJECT CONTEXT
// =============================================================================

/*
 * Context for building ELF/DWARF structures
 *
 * This structure maintains state while constructing DWARF unwind information.
 * It acts as a simple buffer manager with pointers to track current position
 * and important landmarks within the buffer.
 */
typedef struct ELFObjectContext {
    uint8_t* p;            // Current write position in buffer
    uint8_t* startp;       // Start of buffer (for offset calculations)
    uint8_t* fde_p;        // Start of FDE data (for PC-relative calculations)
    uintptr_t code_addr;   // Address of the code section
    size_t code_size;      // Size of the code section
} ELFObjectContext;

// =============================================================================
//                              DWARF GENERATION UTILITIES
// =============================================================================

/*
 * Append a null-terminated string to the ELF context buffer.
 *
 * Args:
 *   ctx: ELF object context
 *   str: String to append (must be null-terminated)
 *
 * Returns: Offset from start of buffer where string was written
 */
static uint32_t elfctx_append_string(ELFObjectContext* ctx, const char* str) {
    uint8_t* p = ctx->p;
    uint32_t ofs = (uint32_t)(p - ctx->startp);

    /* Copy string including null terminator */
    do {
        *p++ = (uint8_t)*str;
    } while (*str++);

    ctx->p = p;
    return ofs;
}

/*
 * Append a SLEB128 (Signed Little Endian Base 128) value
 *
 * SLEB128 is a variable-length encoding used extensively in DWARF.
 * It efficiently encodes small numbers in fewer bytes.
 *
 * Args:
 *   ctx: ELF object context
 *   v: Signed value to encode
 */
static void elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) {
    uint8_t* p = ctx->p;

    /* Encode 7 bits at a time, with continuation bit in MSB */
    for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) {
        *p++ = (uint8_t)((v & 0x7f) | 0x80);  // Set continuation bit
    }
    *p++ = (uint8_t)(v & 0x7f);  // Final byte without continuation bit

    ctx->p = p;
}

/*
 * Append a ULEB128 (Unsigned Little Endian Base 128) value
 *
 * Similar to SLEB128 but for unsigned values.
 *
 * Args:
 *   ctx: ELF object context
 *   v: Unsigned value to encode
 */
static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) {
    uint8_t* p = ctx->p;

    /* Encode 7 bits at a time, with continuation bit in MSB */
    for (; v >= 0x80; v >>= 7) {
        *p++ = (char)((v & 0x7f) | 0x80);  // Set continuation bit
    }
    *p++ = (char)v;  // Final byte without continuation bit

    ctx->p = p;
}

/*
 * Macros for generating DWARF structures
 *
 * These macros provide a convenient way to write various data types
 * to the DWARF buffer while automatically advancing the pointer.
 */
#define DWRF_U8(x) (*p++ = (x))                                    // Write unsigned 8-bit
#define DWRF_I8(x) (*(int8_t*)p = (x), p++)                       // Write signed 8-bit
#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2)                 // Write unsigned 16-bit
#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4)                 // Write unsigned 32-bit
#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) // Write address
#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) // Write ULEB128
#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) // Write SLEB128
#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) // Write string

/* Align to specified boundary with NOP instructions */
#define DWRF_ALIGNNOP(s)                                          \
    while ((uintptr_t)p & ((s)-1)) {                              \
        *p++ = DWRF_CFA_nop;                                       \
    }

/* Write a DWARF section with automatic size calculation */
#define DWRF_SECTION(name, stmt)                                  \
    {                                                             \
        uint32_t* szp_##name = (uint32_t*)p;                      \
        p += 4;                                                   \
        stmt;                                                     \
        *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \
    }

// =============================================================================
//                              DWARF EH FRAME GENERATION
// =============================================================================

static void elf_init_ehframe_perf(ELFObjectContext* ctx);
#if defined(PY_HAVE_JIT_GDB_UNWIND)
static void elf_init_ehframe_gdb(ELFObjectContext* ctx);
#endif

static inline void elf_init_ehframe(ELFObjectContext* ctx, int absolute_addr) {
    if (absolute_addr) {
#if defined(PY_HAVE_JIT_GDB_UNWIND)
        elf_init_ehframe_gdb(ctx);
#else
        Py_UNREACHABLE();
#endif
    }
    else {
        elf_init_ehframe_perf(ctx);
    }
}

size_t
_PyJitUnwind_EhFrameSize(int absolute_addr)
{
    /* The .eh_frame we emit is small and bounded; keep a generous buffer. */
    uint8_t scratch[512];
    _Static_assert(sizeof(scratch) >= 256,
                   "scratch buffer may be too small for elf_init_ehframe");
    ELFObjectContext ctx;
    ctx.code_size = 1;
    ctx.code_addr = 0;
    ctx.startp = ctx.p = scratch;
    ctx.fde_p = NULL;
    /* Generate once into scratch to learn the required size. */
    elf_init_ehframe(&ctx, absolute_addr);
    ptrdiff_t size = ctx.p - ctx.startp;
    assert(size <= (ptrdiff_t)sizeof(scratch));
    return (size_t)size;
}

size_t
_PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size,
                        const void *code_addr, size_t code_size,
                        int absolute_addr)
{
    if (buffer == NULL || code_addr == NULL || code_size == 0) {
        return 0;
    }
    /* Generate the frame twice: once to size-check, once to write. */
    size_t required = _PyJitUnwind_EhFrameSize(absolute_addr);
    if (required == 0 || required > buffer_size) {
        return 0;
    }
    ELFObjectContext ctx;
    ctx.code_size = code_size;
    ctx.code_addr = (uintptr_t)code_addr;
    ctx.startp = ctx.p = buffer;
    ctx.fde_p = NULL;
    elf_init_ehframe(&ctx, absolute_addr);
    size_t written = (size_t)(ctx.p - ctx.startp);
    /* The frame size is independent of code_addr/code_size (fixed-width fields). */
    assert(written == required);
    return written;
}

/*
 * Generate a minimal .eh_frame for a single JIT code region.
 *
 * The .eh_frame section contains Call Frame Information (CFI) that describes
 * how to unwind the stack at any point in the code. This is essential for
 * unwinding through JIT-generated code.
 *
 * The generated data contains:
 * 1. A CIE (Common Information Entry) describing the calling convention.
 * 2. An FDE (Frame Description Entry) describing how to unwind the JIT frame.
 *
 * Two flavors are emitted, dispatched on the absolute_addr flag:
 *
 * - absolute_addr == 0 (elf_init_ehframe_perf): PC-relative FDE address
 *   encoding for perf's synthesized DSO layout. The CIE describes the
 *   trampoline's entry state and the FDE walks through the prologue and
 *   epilogue with advance_loc instructions. This matches the pre-existing
 *   perf_jit_trampoline behavior byte-for-byte.
 *
 * - absolute_addr == 1 (elf_init_ehframe_gdb): absolute FDE address
 *   encoding for the GDB JIT in-memory ELF. The CIE describes the
 *   steady-state frame layout (CFA = %rbp+16 / x29+16, with saved fp and
 *   return-address column at fixed offsets) and the FDE emits no further
 *   CFI. The same rule applies at every PC in the registered region,
 *   which is correct for executor stencils (they pin the frame pointer
 *   across the region). This is the GDB-side fix; see elf_init_ehframe_gdb
 *   for details.
 */
static void elf_init_ehframe_perf(ELFObjectContext* ctx) {
    int fde_ptr_enc = DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4;
    uint8_t* p = ctx->p;
    uint8_t* framep = p;  // Remember start of frame data

    /*
    * DWARF Unwind Table for Trampoline Function
    *
    * This section defines DWARF Call Frame Information (CFI) using encoded macros
    * like `DWRF_U8`, `DWRF_UV`, and `DWRF_SECTION` to describe how the trampoline function
    * preserves and restores registers. This is used by profiling tools (e.g., `perf`)
    * and debuggers for stack unwinding in JIT-compiled code.
    *
    * -------------------------------------------------
    * TO REGENERATE THIS TABLE FROM GCC OBJECTS:
    * -------------------------------------------------
    *
    * 1. Create a trampoline source file (e.g., `trampoline.c`):
    *
    *      #include <Python.h>
    *      typedef PyObject* (*py_evaluator)(void*, void*, int);
    *      PyObject* trampoline(void *ts, void *f, int throwflag, py_evaluator evaluator) {
    *          return evaluator(ts, f, throwflag);
    *      }
    *
    * 2. Compile to an object file with frame pointer preservation:
    *
    *      gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c
    *
    * 3. Extract DWARF unwind info from the object file:
    *
    *      readelf -w trampoline.o
    *
    *    Example output from `.eh_frame`:
    *
    *      00000000 CIE
    *        Version:               1
    *        Augmentation:          "zR"
    *        Code alignment factor: 4
    *        Data alignment factor: -8
    *        Return address column: 30
    *        DW_CFA_def_cfa: r31 (sp) ofs 0
    *
    *      00000014 FDE cie=00000000 pc=0..14
    *        DW_CFA_advance_loc: 4
    *        DW_CFA_def_cfa_offset: 16
    *        DW_CFA_offset: r29 at cfa-16
    *        DW_CFA_offset: r30 at cfa-8
    *        DW_CFA_advance_loc: 12
    *        DW_CFA_restore: r30
    *        DW_CFA_restore: r29
    *        DW_CFA_def_cfa_offset: 0
    *
    * -- These values can be verified by comparing with `readelf -w` or `llvm-dwarfdump --eh-frame`.
    *
    * ----------------------------------
    * HOW TO TRANSLATE TO DWRF_* MACROS:
    * ----------------------------------
    *
    * After compiling your trampoline with:
    *
    *     gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c
    *
    * run:
    *
    *     readelf -w trampoline.o
    *
    * to inspect the generated `.eh_frame` data. You will see two main components:
    *
    *     1. A CIE (Common Information Entry): shared configuration used by all FDEs.
    *     2. An FDE (Frame Description Entry): function-specific unwind instructions.
    *
    * ---------------------
    * Translating the CIE:
    * ---------------------
    * From `readelf -w`, you might see:
    *
    *   00000000 0000000000000010 00000000 CIE
    *     Version:               1
    *     Augmentation:          "zR"
    *     Code alignment factor: 4
    *     Data alignment factor: -8
    *     Return address column: 30
    *     Augmentation data:     1b
    *     DW_CFA_def_cfa: r31 (sp) ofs 0
    *
    * Map this to:
    *
    *     DWRF_SECTION(CIE,
    *         DWRF_U32(0);                             // CIE ID (always 0 for CIEs)
    *         DWRF_U8(DWRF_CIE_VERSION);              // Version: 1
    *         DWRF_STR("zR");                         // Augmentation string "zR"
    *         DWRF_UV(4);                             // Code alignment factor = 4
    *         DWRF_SV(-8);                            // Data alignment factor = -8
    *         DWRF_U8(DWRF_REG_RA);                   // Return address register (e.g., x30 = 30)
    *         DWRF_UV(1);                             // Augmentation data length = 1
    *         DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // Encoding for FDE pointers
    *
    *         DWRF_U8(DWRF_CFA_def_cfa);              // DW_CFA_def_cfa
    *         DWRF_UV(DWRF_REG_SP);                   // Register: SP (r31)
    *         DWRF_UV(0);                             // Offset = 0
    *
    *         DWRF_ALIGNNOP(sizeof(uintptr_t));       // Align to pointer size boundary
    *     )
    *
    * Notes:
    *   - Use `DWRF_UV` for unsigned LEB128, `DWRF_SV` for signed LEB128.
    *   - `DWRF_REG_RA` and `DWRF_REG_SP` are architecture-defined constants.
    *
    * ---------------------
    * Translating the FDE:
    * ---------------------
    * From `readelf -w`:
    *
    *   00000014 0000000000000020 00000018 FDE cie=00000000 pc=0000000000000000..0000000000000014
    *     DW_CFA_advance_loc: 4
    *     DW_CFA_def_cfa_offset: 16
    *     DW_CFA_offset: r29 at cfa-16
    *     DW_CFA_offset: r30 at cfa-8
    *     DW_CFA_advance_loc: 12
    *     DW_CFA_restore: r30
    *     DW_CFA_restore: r29
    *     DW_CFA_def_cfa_offset: 0
    *
    * Map the FDE header and instructions to:
    *
    *     DWRF_SECTION(FDE,
    *         DWRF_U32((uint32_t)(p - framep));       // Offset to CIE (relative from here)
    *         DWRF_U32(pc_relative_offset);           // PC-relative location of the code (calculated dynamically)
    *         DWRF_U32(ctx->code_size);               // Code range covered by this FDE
    *         DWRF_U8(0);                             // Augmentation data length (none)
    *
    *         DWRF_U8(DWRF_CFA_advance_loc | 1);      // Advance location by 1 unit (1 * 4 = 4 bytes)
    *         DWRF_U8(DWRF_CFA_def_cfa_offset);       // CFA = SP + 16
    *         DWRF_UV(16);
    *
    *         DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Save x29 (frame pointer)
    *         DWRF_UV(2);                             // At offset 2 * 8 = 16 bytes
    *
    *         DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Save x30 (return address)
    *         DWRF_UV(1);                             // At offset 1 * 8 = 8 bytes
    *
    *         DWRF_U8(DWRF_CFA_advance_loc | 3);      // Advance location by 3 units (3 * 4 = 12 bytes)
    *
    *         DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore x30
    *         DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore x29
    *
    *         DWRF_U8(DWRF_CFA_def_cfa_offset);       // CFA = SP
    *         DWRF_UV(0);
    *     )
    *
    * To regenerate:
    *   1. Get the `code alignment factor`, `data alignment factor`, and `RA column` from the CIE.
    *   2. Note the range of the function from the FDE's `pc=...` line and map it to the JIT code as
    *      the code is in a different address space every time.
    *   3. For each `DW_CFA_*` entry, use the corresponding `DWRF_*` macro:
    *        - `DW_CFA_def_cfa_offset`     → DWRF_U8(DWRF_CFA_def_cfa_offset), DWRF_UV(value)
    *        - `DW_CFA_offset: rX`         → DWRF_U8(DWRF_CFA_offset | reg), DWRF_UV(offset)
    *        - `DW_CFA_restore: rX`        → DWRF_U8(DWRF_CFA_offset | reg) // restore is same as reusing offset
    *        - `DW_CFA_advance_loc: N`     → DWRF_U8(DWRF_CFA_advance_loc | (N / code_alignment_factor))
    *   4. Use `DWRF_REG_FP`, `DWRF_REG_RA`, etc., for register numbers.
    *   5. Use `sizeof(uintptr_t)` (typically 8) for pointer size calculations and alignment.
    */

    /*
     * Emit DWARF EH CIE (Common Information Entry)
     *
     * The CIE describes the calling conventions and basic unwinding rules
     * that apply to all functions in this compilation unit.
     */
    DWRF_SECTION(CIE,
        DWRF_U32(0);                           // CIE ID (0 indicates this is a CIE)
        DWRF_U8(DWRF_CIE_VERSION);            // CIE version (1)
        DWRF_STR("zR");                       // Augmentation string ("zR" = has LSDA)
#ifdef __x86_64__
        DWRF_UV(1);                           // Code alignment factor (x86_64: 1 byte)
#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__)
        DWRF_UV(4);                           // Code alignment factor (AArch64: 4 bytes per instruction)
#endif
        DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative)
        DWRF_U8(DWRF_REG_RA);                 // Return address register number
        DWRF_UV(1);                           // Augmentation data length
        DWRF_U8(fde_ptr_enc);                 // FDE pointer encoding

        /* Initial CFI instructions - describe default calling convention */
#ifdef __x86_64__
        /* x86_64 initial CFI state */
        DWRF_U8(DWRF_CFA_def_cfa);            // Define CFA (Call Frame Address)
        DWRF_UV(DWRF_REG_SP);                 // CFA = SP register
        DWRF_UV(sizeof(uintptr_t));           // CFA = SP + pointer_size
        DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved
        DWRF_UV(1);                           // At offset 1 from CFA
#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__)
        /* AArch64 initial CFI state */
        DWRF_U8(DWRF_CFA_def_cfa);            // Define CFA (Call Frame Address)
        DWRF_UV(DWRF_REG_SP);                 // CFA = SP register
        DWRF_UV(0);                           // CFA = SP + 0 (AArch64 starts with offset 0)
        // No initial register saves in AArch64 CIE
#endif
        DWRF_ALIGNNOP(sizeof(uintptr_t));     // Align to pointer boundary
    )

    /*
     * Emit DWARF EH FDE (Frame Description Entry)
     *
     * The FDE describes unwinding information specific to this function.
     * It references the CIE and provides function-specific CFI instructions.
     *
     * The PC-relative offset is calculated after the entire EH frame is built
     * to ensure accurate positioning relative to the synthesized DSO layout.
     */
    DWRF_SECTION(FDE,
        DWRF_U32((uint32_t)(p - framep));     // Offset to CIE (backwards reference)
        /*
         * In perf jitdump mode the FDE PC field is encoded PC-relative and
         * points back to code_start. Record where that field lives so we can
         * patch in the final offset after the rest of the synthetic DSO
         * layout is known.
         */
        ctx->fde_p = p;                       // Remember where PC offset field is located for later calculation
        DWRF_U32(0);                          // Placeholder for PC-relative offset (calculated below)
        DWRF_U32(ctx->code_size);             // Address range covered by this FDE (code length)
        DWRF_U8(0);                           // Augmentation data length (none)

        /*
         * Architecture-specific CFI instructions
         *
         * These instructions describe how registers are saved and restored
         * during function calls. Each architecture has different calling
         * conventions and register usage patterns.
         */
#ifdef __x86_64__
        /* x86_64 calling convention unwinding rules */
#  if defined(__CET__) && (__CET__ & 1)
        DWRF_U8(DWRF_CFA_advance_loc | 4);    // Advance past endbr64 (4 bytes)
#  endif
        DWRF_U8(DWRF_CFA_advance_loc | 1);    // Advance past push %rbp (1 byte)
        DWRF_U8(DWRF_CFA_def_cfa_offset);     // def_cfa_offset 16
        DWRF_UV(16);                          // New offset: SP + 16
        DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16
        DWRF_UV(2);                           // Offset factor: 2 * 8 = 16 bytes
        DWRF_U8(DWRF_CFA_advance_loc | 3);    // Advance past mov %rsp,%rbp (3 bytes)
        DWRF_U8(DWRF_CFA_def_cfa_register);   // def_cfa_register r6
        DWRF_UV(DWRF_REG_BP);                 // Use base pointer register
        DWRF_U8(DWRF_CFA_advance_loc | 3);    // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3
        DWRF_U8(DWRF_CFA_def_cfa);            // def_cfa r7 ofs 8
        DWRF_UV(DWRF_REG_SP);                 // Use stack pointer register
        DWRF_UV(8);                           // New offset: SP + 8
#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__)
        /* AArch64 calling convention unwinding rules */
        DWRF_U8(DWRF_CFA_advance_loc | 1);        // Advance by 1 instruction (4 bytes)
        DWRF_U8(DWRF_CFA_def_cfa_offset);         // CFA = SP + 16
        DWRF_UV(16);                              // Stack pointer moved by 16 bytes
        DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP);   // x29 (frame pointer) saved
        DWRF_UV(2);                               // At CFA-16 (2 * 8 = 16 bytes from CFA)
        DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA);   // x30 (link register) saved
        DWRF_UV(1);                               // At CFA-8 (1 * 8 = 8 bytes from CFA)
        DWRF_U8(DWRF_CFA_advance_loc | 3);        // Advance by 3 instructions (12 bytes)
        DWRF_U8(DWRF_CFA_def_cfa_register);       // CFA = FP (x29) + 16
        DWRF_UV(DWRF_REG_FP);
        DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA);  // Restore x30 - NO DWRF_UV() after this!
        DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP);  // Restore x29 - NO DWRF_UV() after this!
        DWRF_U8(DWRF_CFA_def_cfa);                // CFA = SP + 0 (stack restored)
        DWRF_UV(DWRF_REG_SP);
        DWRF_UV(0);

#else
#    error "Unsupported target architecture"
#endif

        DWRF_ALIGNNOP(sizeof(uintptr_t));     // Align to pointer boundary
    )

    ctx->p = p;  // Update context pointer to end of generated data

    /* Calculate and update the PC-relative offset in the FDE
     *
     * When perf processes the jitdump, it creates a synthesized DSO with this layout:
     *
     *     Synthesized DSO Memory Layout:
     *     ┌─────────────────────────────────────────────────────────────┐ < code_start
     *     │                        Code Section                         │
     *     │                    (round_up(code_size, 8) bytes)           │
     *     ├─────────────────────────────────────────────────────────────┤ < start of EH frame data
     *     │                      EH Frame Data                          │
     *     │  ┌─────────────────────────────────────────────────────┐    │
     *     │  │                 CIE data                            │    │
     *     │  └─────────────────────────────────────────────────────┘    │
     *     │  ┌─────────────────────────────────────────────────────┐    │
     *     │  │ FDE Header:                                         │    │
     *     │  │   - CIE offset (4 bytes)                            │    │
     *     │  │   - PC offset (4 bytes) <─ fde_offset_in_frame ─────┼────┼─> points to code_start
     *     │  │   - address range (4 bytes)                         │    │   (this specific field)
     *     │  │ CFI Instructions...                                 │    │
     *     │  └─────────────────────────────────────────────────────┘    │
     *     ├─────────────────────────────────────────────────────────────┤ < reference_point
     *     │                    EhFrameHeader                            │
     *     │                 (navigation metadata)                       │
     *     └─────────────────────────────────────────────────────────────┘
     *
     * The PC offset field in the FDE must contain the distance from itself to code_start:
     *
     *   distance = code_start - fde_pc_field
     *
     * Where:
     *   fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame
     *   code_start_location = reference_point - eh_frame_size - round_up(code_size, 8)
     *
     * Therefore:
     *   distance = code_start_location - fde_pc_field_location
     *            = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame)
     *            = -rounded_code_size - fde_offset_in_frame
     *            = -(round_up(code_size, 8) + fde_offset_in_frame)
     *
     * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field.
     *
     */
    int32_t rounded_code_size =
        (int32_t)_Py_SIZE_ROUND_UP(ctx->code_size, 8);
    int32_t fde_offset_in_frame = (int32_t)(ctx->fde_p - framep);
    *(int32_t *)ctx->fde_p = -(rounded_code_size + fde_offset_in_frame);
}

/*
 * Build .eh_frame data for the GDB JIT interface.
 *
 * The executor runs inside the frame established by _PyJIT_Entry, but the
 * synthetic executor FDE collapses that state into a single logical JIT frame
 * that unwinds directly into _PyEval_*. Executor stencils never touch the
 * frame pointer - enforced by Tools/jit/_optimizers.py _validate() and
 * -mframe-pointer=reserved - so the steady-state rule is valid at every PC
 * and the FDE body is empty. Tools/jit/_targets.py derives the initial CFI
 * rules from the row active at the executor call in the compiled shim object.
 */
#if defined(PY_HAVE_JIT_GDB_UNWIND)
static void elf_init_ehframe_gdb(ELFObjectContext* ctx) {
    int fde_ptr_enc = DWRF_EH_PE_absptr;
    uint8_t* p = ctx->p;
    uint8_t* framep = p;

    DWRF_SECTION(CIE,
        DWRF_U32(0);                          // CIE ID
        DWRF_U8(DWRF_CIE_VERSION);
        DWRF_STR("zR");                       // aug data length + FDE ptr encoding follow
        DWRF_UV(JIT_UNWIND_CODE_ALIGNMENT_FACTOR);
        DWRF_SV(JIT_UNWIND_DATA_ALIGNMENT_FACTOR);
        DWRF_U8(JIT_UNWIND_RA_REG);
        DWRF_UV(1);                           // Augmentation data length
        DWRF_U8(fde_ptr_enc);                 // FDE pointer encoding

        /* Executor steady-state rule (our invariant, not the compiler's). */
        DWRF_U8(DWRF_CFA_def_cfa);
        DWRF_UV(JIT_UNWIND_CFA_REG);
        DWRF_UV(JIT_UNWIND_CFA_OFFSET);
        DWRF_U8(DWRF_CFA_offset | JIT_UNWIND_FP_REG);
        DWRF_UV(JIT_UNWIND_FP_OFFSET);
        DWRF_U8(DWRF_CFA_offset | JIT_UNWIND_RA_REG);
        DWRF_UV(JIT_UNWIND_RA_OFFSET);
        DWRF_ALIGNNOP(sizeof(uintptr_t));
    )

    DWRF_SECTION(FDE,
        DWRF_U32((uint32_t)(p - framep));     // Offset to CIE (backwards reference)
        DWRF_ADDR(ctx->code_addr);            // Absolute code start
        DWRF_ADDR((uintptr_t)ctx->code_size); // Code range covered
        DWRF_U8(0);                           // Augmentation data length (none)
        DWRF_ALIGNNOP(sizeof(uintptr_t));
    )

    ctx->p = p;
}
#endif

#if defined(PY_HAVE_JIT_GDB_UNWIND)
enum {
    JIT_NOACTION = 0,
    JIT_REGISTER_FN = 1,
    JIT_UNREGISTER_FN = 2,
};

struct jit_code_entry {
    struct jit_code_entry *next;
    struct jit_code_entry *prev;
    const char *symfile_addr;
    uint64_t symfile_size;
    const void *code_addr;
};

struct jit_descriptor {
    uint32_t version;
    uint32_t action_flag;
    struct jit_code_entry *relevant_entry;
    struct jit_code_entry *first_entry;
};

PyMutex _Py_jit_debug_mutex = {0};

Py_EXPORTED_SYMBOL volatile struct jit_descriptor __jit_debug_descriptor = {
    1, JIT_NOACTION, NULL, NULL
};

Py_EXPORTED_SYMBOL void __attribute__((noinline))
__jit_debug_register_code(void)
{
    /* Keep this call visible to debuggers and not optimized away. */
    (void)__jit_debug_descriptor.action_flag;
#if defined(__GNUC__) || defined(__clang__)
    __asm__ __volatile__("" ::: "memory");
#endif
}

static uint16_t
gdb_jit_machine_id(void)
{
    /* Map the current target to ELF e_machine; return 0 to skip registration. */
#if defined(__x86_64__) || defined(_M_X64)
    return EM_X86_64;
#elif defined(__aarch64__) && !defined(__ILP32__)
    return EM_AARCH64;
#else
    return 0;
#endif
}

static struct jit_code_entry *
gdb_jit_register_code(
    const void *code_addr,
    size_t code_size,
    const char *symname,
    const uint8_t *eh_frame,
    size_t eh_frame_size
)
{
    /*
     * Build a minimal in-memory ELF for GDB's JIT interface and link it into
     * __jit_debug_descriptor so debuggers can resolve JIT code.
     */
    if (code_addr == NULL || code_size == 0 || symname == NULL) {
        return NULL;
    }

    const uint16_t machine = gdb_jit_machine_id();
    if (machine == 0) {
        return NULL;
    }

    enum {
        SH_NULL = 0,
        SH_TEXT,
        SH_EH_FRAME,
        SH_SHSTRTAB,
        SH_STRTAB,
        SH_SYMTAB,
        SH_NUM,
    };
    static const char shstrtab[] =
        "\0.text\0.eh_frame\0.shstrtab\0.strtab\0.symtab";
    _Static_assert(sizeof(shstrtab) ==
        1 + sizeof(".text") + sizeof(".eh_frame") +
            sizeof(".shstrtab") + sizeof(".strtab") + sizeof(".symtab"),
        "shstrtab size mismatch");
    const size_t shstrtab_size = sizeof(shstrtab);
    const size_t sh_text = 1;
    const size_t sh_eh_frame = sh_text + sizeof(".text");
    const size_t sh_shstrtab = sh_eh_frame + sizeof(".eh_frame");
    const size_t sh_strtab = sh_shstrtab + sizeof(".shstrtab");
    const size_t sh_symtab = sh_strtab + sizeof(".strtab");
    const size_t text_size = code_size;
    const size_t text_padded = _Py_SIZE_ROUND_UP(text_size, 8);
    const size_t strtab_size = 1 + strlen(symname) + 1;
    const size_t symtab_size = 3 * sizeof(Elf64_Sym);

    size_t offset = sizeof(Elf64_Ehdr);
    offset = _Py_SIZE_ROUND_UP(offset, 16);
    const size_t text_off = offset;
    const size_t eh_off = text_off + text_padded;
    offset = eh_off + eh_frame_size;
    const size_t shstr_off = offset;
    offset += shstrtab_size;
    const size_t str_off = offset;
    offset += strtab_size;
    /* Elf64_Sym requires 8-byte alignment for st_value/st_size. */
    offset = _Py_SIZE_ROUND_UP(offset, 8);
    const size_t sym_off = offset;
    offset += symtab_size;
    offset = _Py_SIZE_ROUND_UP(offset, sizeof(Elf64_Shdr));
    const size_t sh_off = offset;

    const size_t shnum = SH_NUM;
    const size_t total_size = sh_off + shnum * sizeof(Elf64_Shdr);
    uint8_t *buf = (uint8_t *)PyMem_RawMalloc(total_size);
    if (buf == NULL) {
        return NULL;
    }
    memset(buf, 0, total_size);

    Elf64_Ehdr *ehdr = (Elf64_Ehdr *)buf;
    memcpy(ehdr->e_ident, ELFMAG, SELFMAG);
    ehdr->e_ident[EI_CLASS] = ELFCLASS64;
    ehdr->e_ident[EI_DATA] = ELFDATA2LSB;
    ehdr->e_ident[EI_VERSION] = EV_CURRENT;
    ehdr->e_ident[EI_OSABI] = ELFOSABI_NONE;
    ehdr->e_type = ET_DYN;
    ehdr->e_machine = machine;
    ehdr->e_version = EV_CURRENT;
    ehdr->e_entry = 0;
    ehdr->e_phoff = 0;
    ehdr->e_shoff = sh_off;
    ehdr->e_ehsize = sizeof(Elf64_Ehdr);
    ehdr->e_shentsize = sizeof(Elf64_Shdr);
    ehdr->e_shnum = shnum;
    ehdr->e_shstrndx = SH_SHSTRTAB;

    memcpy(buf + text_off, code_addr, text_size);
    memcpy(buf + eh_off, eh_frame, eh_frame_size);

    char *shstr = (char *)(buf + shstr_off);
    memcpy(shstr, shstrtab, shstrtab_size);

    char *strtab = (char *)(buf + str_off);
    strtab[0] = '\0';
    memcpy(strtab + 1, symname, strlen(symname));
    strtab[strtab_size - 1] = '\0';

    Elf64_Sym *syms = (Elf64_Sym *)(buf + sym_off);
    memset(syms, 0, symtab_size);
    /* Section symbol for .text (local) */
    syms[1].st_info = ELF64_ST_INFO(STB_LOCAL, STT_SECTION);
    syms[1].st_shndx = 1;
    /* Function symbol */
    syms[2].st_name = 1;
    syms[2].st_info = ELF64_ST_INFO(STB_GLOBAL, STT_FUNC);
    syms[2].st_other = STV_DEFAULT;
    syms[2].st_shndx = 1;
    /* For ET_DYN/ET_EXEC, st_value is the absolute virtual address. */
    syms[2].st_value = (Elf64_Addr)(uintptr_t)code_addr;
    syms[2].st_size = code_size;

    Elf64_Shdr *shdrs = (Elf64_Shdr *)(buf + sh_off);
    memset(shdrs, 0, shnum * sizeof(Elf64_Shdr));

    shdrs[SH_TEXT].sh_name = sh_text;
    shdrs[SH_TEXT].sh_type = SHT_PROGBITS;
    shdrs[SH_TEXT].sh_flags = SHF_ALLOC | SHF_EXECINSTR;
    shdrs[SH_TEXT].sh_addr = (Elf64_Addr)(uintptr_t)code_addr;
    shdrs[SH_TEXT].sh_offset = text_off;
    shdrs[SH_TEXT].sh_size = text_size;
    shdrs[SH_TEXT].sh_addralign = 16;

    shdrs[SH_EH_FRAME].sh_name = sh_eh_frame;
    shdrs[SH_EH_FRAME].sh_type = SHT_PROGBITS;
    shdrs[SH_EH_FRAME].sh_flags = SHF_ALLOC;
    shdrs[SH_EH_FRAME].sh_addr =
        (Elf64_Addr)((uintptr_t)code_addr + text_padded);
    shdrs[SH_EH_FRAME].sh_offset = eh_off;
    shdrs[SH_EH_FRAME].sh_size = eh_frame_size;
    shdrs[SH_EH_FRAME].sh_addralign = 8;

    shdrs[SH_SHSTRTAB].sh_name = sh_shstrtab;
    shdrs[SH_SHSTRTAB].sh_type = SHT_STRTAB;
    shdrs[SH_SHSTRTAB].sh_offset = shstr_off;
    shdrs[SH_SHSTRTAB].sh_size = shstrtab_size;
    shdrs[SH_SHSTRTAB].sh_addralign = 1;

    shdrs[SH_STRTAB].sh_name = sh_strtab;
    shdrs[SH_STRTAB].sh_type = SHT_STRTAB;
    shdrs[SH_STRTAB].sh_offset = str_off;
    shdrs[SH_STRTAB].sh_size = strtab_size;
    shdrs[SH_STRTAB].sh_addralign = 1;

    shdrs[SH_SYMTAB].sh_name = sh_symtab;
    shdrs[SH_SYMTAB].sh_type = SHT_SYMTAB;
    shdrs[SH_SYMTAB].sh_offset = sym_off;
    shdrs[SH_SYMTAB].sh_size = symtab_size;
    shdrs[SH_SYMTAB].sh_link = SH_STRTAB;
    shdrs[SH_SYMTAB].sh_info = 2;
    shdrs[SH_SYMTAB].sh_addralign = 8;
    shdrs[SH_SYMTAB].sh_entsize = sizeof(Elf64_Sym);

    struct jit_code_entry *entry = PyMem_RawMalloc(sizeof(*entry));
    if (entry == NULL) {
        PyMem_RawFree(buf);
        return NULL;
    }
    entry->symfile_addr = (const char *)buf;
    entry->symfile_size = total_size;
    entry->code_addr = code_addr;

    PyMutex_Lock(&_Py_jit_debug_mutex);
    entry->prev = NULL;
    entry->next = __jit_debug_descriptor.first_entry;
    if (entry->next != NULL) {
        entry->next->prev = entry;
    }
    __jit_debug_descriptor.first_entry = entry;
    __jit_debug_descriptor.relevant_entry = entry;
    __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
    __jit_debug_register_code();
    __jit_debug_descriptor.action_flag = JIT_NOACTION;
    __jit_debug_descriptor.relevant_entry = NULL;
    PyMutex_Unlock(&_Py_jit_debug_mutex);
    return entry;
}
#endif  // defined(PY_HAVE_JIT_GDB_UNWIND)

void *
_PyJitUnwind_GdbRegisterCode(const void *code_addr,
                             size_t code_size,
                             const char *entry,
                             const char *filename)
{
#if defined(PY_HAVE_JIT_GDB_UNWIND)
    /* GDB expects a stable symbol name and absolute addresses in .eh_frame. */
    if (entry == NULL) {
        entry = "";
    }
    if (filename == NULL) {
        filename = "";
    }
    size_t name_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1;
    char *name = (char *)PyMem_RawMalloc(name_size);
    if (name == NULL) {
        return NULL;
    }
    snprintf(name, name_size, "py::%s:%s", entry, filename);

    uint8_t buffer[1024];
    size_t eh_frame_size = _PyJitUnwind_BuildEhFrame(
        buffer, sizeof(buffer), code_addr, code_size, 1);
    if (eh_frame_size == 0) {
        PyMem_RawFree(name);
        return NULL;
    }

    void *handle = gdb_jit_register_code(code_addr, code_size, name,
                                         buffer, eh_frame_size);
    PyMem_RawFree(name);
    return handle;
#else
    (void)code_addr;
    (void)code_size;
    (void)entry;
    (void)filename;
    return NULL;
#endif
}

void
_PyJitUnwind_GdbUnregisterCode(void *handle)
{
#if defined(PY_HAVE_JIT_GDB_UNWIND)
    struct jit_code_entry *entry = (struct jit_code_entry *)handle;
    if (entry == NULL) {
        return;
    }

    PyMutex_Lock(&_Py_jit_debug_mutex);
    if (entry->prev != NULL) {
        entry->prev->next = entry->next;
    }
    else {
        __jit_debug_descriptor.first_entry = entry->next;
    }
    if (entry->next != NULL) {
        entry->next->prev = entry->prev;
    }

    __jit_debug_descriptor.relevant_entry = entry;
    __jit_debug_descriptor.action_flag = JIT_UNREGISTER_FN;
    __jit_debug_register_code();
    __jit_debug_descriptor.action_flag = JIT_NOACTION;
    __jit_debug_descriptor.relevant_entry = NULL;

    PyMutex_Unlock(&_Py_jit_debug_mutex);

    PyMem_RawFree((void *)entry->symfile_addr);
    PyMem_RawFree(entry);
#else
    (void)handle;
#endif
}

#if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND)
void *
_PyJitUnwind_GnuBacktraceRegisterCode(const void *code_addr, size_t code_size)
{
    if (code_addr == NULL || code_size == 0) {
        return NULL;
    }

    size_t eh_frame_size = _PyJitUnwind_EhFrameSize(1);
    if (eh_frame_size == 0) {
        return NULL;
    }
    size_t total_size = eh_frame_size + sizeof(uint32_t);
    if (total_size < eh_frame_size) {
        return NULL;
    }

    /*
     * libgcc's __register_frame walks a .eh_frame section until it finds a
     * zero-length terminator entry, so keep an extra zeroed word after the
     * generated CIE/FDE pair.
     *
     * See GCC's libgcc/unwind-dw2-fde.c (__register_frame) and
     * libgcc/unwind-dw2-fde.h (last_fde/next_fde):
     * https://github.com/gcc-mirror/gcc/blob/master/libgcc/unwind-dw2-fde.c
     * https://github.com/gcc-mirror/gcc/blob/master/libgcc/unwind-dw2-fde.h
     */
    uint8_t *eh_frame = PyMem_RawCalloc(1, total_size);
    if (eh_frame == NULL) {
        return NULL;
    }
    if (_PyJitUnwind_BuildEhFrame(
            eh_frame, eh_frame_size, code_addr, code_size, 1) == 0) {
        PyMem_RawFree(eh_frame);
        return NULL;
    }

    __register_frame(eh_frame);
    return eh_frame;
}

void
_PyJitUnwind_GnuBacktraceUnregisterCode(void *handle)
{
    if (handle == NULL) {
        return;
    }
    __deregister_frame(handle);
    PyMem_RawFree(handle);
}
#endif  // defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND)

#endif  // JIT unwind support