Source/Essentials/MemorySystem.cpp - master - Mr.Stalin/FOnline-Engine

Source/Essentials/MemorySystem.cpp
644 строки15 KB
cvet
Alloc fixes (#192)
27 июл 2026, 14:30
27 июл 2026, 14:30ba3760e
//      __________        ___               ______            _
//     / ____/ __ \____  / (_)___  ___     / ____/___  ____ _(_)___  ___
//    / /_  / / / / __ \/ / / __ \/ _ \   / __/ / __ \/ __ `/ / __ \/ _ `
//   / __/ / /_/ / / / / / / / / /  __/  / /___/ / / / /_/ / / / / /  __/
//  /_/    \____/_/ /_/_/_/_/ /_/\___/  /_____/_/ /_/\__, /_/_/ /_/\___/
//                                                  /____/
// FOnline Engine
// https://fonline.ru
// https://github.com/cvet/fonline
//
// MIT License
//
// Copyright (c) 2006 - 2026, Anton Tsvetinskiy aka cvet <cvet@tut.by>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//

#include "MemorySystem.h"
#include "BaseLogging.h"
#include "GlobalData.h"
#include "StackTrace.h"

// Only the non-rpmalloc Windows path needs the CRT aligned allocation entry points
#if !FO_HAVE_RPMALLOC && FO_WINDOWS
#include <malloc.h>
#endif

FO_BEGIN_NAMESPACE

struct MemorySystemData
{
    MemorySystemData() { InitBackupMemoryChunks(); }

    BadAllocCallback Callback {};
};
FO_GLOBAL_DATA(MemorySystemData, MemorySystem);

// Unpoliced allocation primitives backing the SafeAlloc::*Raw tier. They report failure by returning null
// and are deliberately not part of the public header: every caller must go through SafeAlloc so that an
// allocation failure follows the engine out-of-memory contract instead of being silently propagated.
static auto MemMalloc(size_t size) noexcept -> nptr<void>;
static auto MemCalloc(size_t num, size_t size) noexcept -> nptr<void>;
static auto MemRealloc(nptr<void> ptr, size_t size) noexcept -> nptr<void>;
static void MemFree(nptr<void> ptr) noexcept;
static auto MemAlignedMalloc(size_t size, size_t alignment) noexcept -> nptr<void>;
static void MemAlignedFree(nptr<void> ptr) noexcept;

static constexpr size_t BACKUP_MEMORY_CHUNKS = 100;
static constexpr size_t BACKUP_MEMORY_CHUNK_SIZE = 100000; // 100 chunks x 100kb = 10mb
static unique_arr_ptr<unique_arr_ptr<uint8_t>> BackupMemoryChunks;
static std::atomic_size_t BackupMemoryChunksCount;

// Replace memory allocator
#if FO_HAVE_RPMALLOC

FO_END_NAMESPACE

#if FO_TRACY
#include "client/tracy_rpmalloc.hpp"
#else
#include "rpmalloc.h"
#endif

#include <new>

#if FO_WINDOWS
#define CRTDECL __CRTDECL
#else
#define CRTDECL
#endif

extern void CRTDECL operator delete(void* p) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_TRACY
    TracyFree(p);
    tracy::rpfree(p);
#else
    rpfree(p);
#endif
}

extern void CRTDECL operator delete[](void* p) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_TRACY
    TracyFree(p);
    tracy::rpfree(p);
#else
    rpfree(p);
#endif
}

extern void* CRTDECL operator new(std::size_t size) noexcept(false)
{
    FO_NO_STACK_TRACE_ENTRY();

    void* p = nullptr;
#if FO_TRACY
    tracy::InitRpmalloc();
    p = tracy::rpmalloc(size);
    TracyAlloc(p, size);
#else
    p = rpmalloc(size);
#endif
    if (p == nullptr) {
        throw std::bad_alloc();
    }
    return p;
}

extern void* CRTDECL operator new[](std::size_t size) noexcept(false)
{
    FO_NO_STACK_TRACE_ENTRY();

    void* p = nullptr;
#if FO_TRACY
    tracy::InitRpmalloc();
    p = tracy::rpmalloc(size);
    TracyAlloc(p, size);
#else
    p = rpmalloc(size);
#endif
    if (p == nullptr) {
        throw std::bad_alloc();
    }
    return p;
}

extern void* CRTDECL operator new(std::size_t size, const std::nothrow_t& /*tag*/) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

    void* p = nullptr;
#if FO_TRACY
    tracy::InitRpmalloc();
    p = tracy::rpmalloc(size);
    TracyAlloc(p, size);
#else
    p = rpmalloc(size);
#endif
    return p;
}

extern void* CRTDECL operator new[](std::size_t size, const std::nothrow_t& /*tag*/) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

    void* p = nullptr;
#if FO_TRACY
    tracy::InitRpmalloc();
    p = tracy::rpmalloc(size);
    TracyAlloc(p, size);
#else
    p = rpmalloc(size);
#endif
    return p;
}

extern void CRTDECL operator delete(void* p, std::size_t /*size*/) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_TRACY
    TracyFree(p);
    tracy::rpfree(p);
#else
    rpfree(p);
#endif
}

extern void CRTDECL operator delete[](void* p, std::size_t /*size*/) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_TRACY
    TracyFree(p);
    tracy::rpfree(p);
#else
    rpfree(p);
#endif
}

extern void CRTDECL operator delete(void* p, std::align_val_t /*align*/) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_TRACY
    TracyFree(p);
    tracy::rpfree(p);
#else
    rpfree(p);
#endif
}

extern void CRTDECL operator delete[](void* p, std::align_val_t /*align*/) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_TRACY
    TracyFree(p);
    tracy::rpfree(p);
#else
    rpfree(p);
#endif
}

extern void CRTDECL operator delete(void* p, std::size_t /*size*/, std::align_val_t /*align*/) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_TRACY
    TracyFree(p);
    tracy::rpfree(p);
#else
    rpfree(p);
#endif
}

extern void CRTDECL operator delete[](void* p, std::size_t /*size*/, std::align_val_t /*align*/) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_TRACY
    TracyFree(p);
    tracy::rpfree(p);
#else
    rpfree(p);
#endif
}

extern void* CRTDECL operator new(std::size_t size, std::align_val_t align) noexcept(false)
{
    FO_NO_STACK_TRACE_ENTRY();

    void* p = nullptr;
#if FO_TRACY
    tracy::InitRpmalloc();
    p = tracy::rpaligned_alloc(static_cast<size_t>(align), size);
    TracyAlloc(p, size);
#else
    p = rpaligned_alloc(static_cast<size_t>(align), size);
#endif
    if (p == nullptr) {
        throw std::bad_alloc();
    }
    return p;
}

extern void* CRTDECL operator new[](std::size_t size, std::align_val_t align) noexcept(false)
{
    FO_NO_STACK_TRACE_ENTRY();

    void* p = nullptr;
#if FO_TRACY
    tracy::InitRpmalloc();
    p = tracy::rpaligned_alloc(static_cast<size_t>(align), size);
    TracyAlloc(p, size);
#else
    p = rpaligned_alloc(static_cast<size_t>(align), size);
#endif
    if (p == nullptr) {
        throw std::bad_alloc();
    }
    return p;
}

extern void* CRTDECL operator new(std::size_t size, std::align_val_t align, const std::nothrow_t& /*tag*/) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

    void* p = nullptr;
#if FO_TRACY
    tracy::InitRpmalloc();
    p = tracy::rpaligned_alloc(static_cast<size_t>(align), size);
    TracyAlloc(p, size);
#else
    p = rpaligned_alloc(static_cast<size_t>(align), size);
#endif
    return p;
}

extern void* CRTDECL operator new[](std::size_t size, std::align_val_t align, const std::nothrow_t& /*tag*/) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

    void* p = nullptr;
#if FO_TRACY
    tracy::InitRpmalloc();
    p = tracy::rpaligned_alloc(static_cast<size_t>(align), size);
    TracyAlloc(p, size);
#else
    p = rpaligned_alloc(static_cast<size_t>(align), size);
#endif
    return p;
}

#undef CRTDECL
FO_BEGIN_NAMESPACE

#endif

auto SafeAlloc::MallocRaw(size_t size) noexcept -> nptr<void>
{
    FO_NO_STACK_TRACE_ENTRY();

    nptr<void> mem = MemMalloc(size);

    if (!mem && size != 0) {
        ReportBadAlloc("Raw malloc failed", "byte", 1, size);

        while (!mem && FreeBackupMemoryChunk()) {
            mem = MemMalloc(size);
        }

        if (!mem) {
            ReportAndExit("Failed to allocate raw from backup pool");
        }
    }

    return mem;
}

auto SafeAlloc::CallocRaw(size_t num, size_t size) noexcept -> nptr<void>
{
    FO_NO_STACK_TRACE_ENTRY();

    if (size != 0 && num > std::numeric_limits<size_t>::max() / size) {
        ReportBadAlloc("Raw calloc size overflow", "byte", num, size);
        ReportAndExit("Raw calloc size overflow");
    }

    nptr<void> mem = MemCalloc(num, size);

    if (!mem && num != 0 && size != 0) {
        ReportBadAlloc("Raw calloc failed", "byte", num, size);

        while (!mem && FreeBackupMemoryChunk()) {
            mem = MemCalloc(num, size);
        }

        if (!mem) {
            ReportAndExit("Failed to allocate raw zeroed from backup pool");
        }
    }

    return mem;
}

auto SafeAlloc::ReallocRaw(nptr<void> ptr, size_t size) noexcept -> nptr<void>
{
    FO_NO_STACK_TRACE_ENTRY();

    nptr<void> mem = MemRealloc(ptr, size);

    if (!mem && size != 0) {
        ReportBadAlloc("Raw realloc failed", "byte", 1, size);

        while (!mem && FreeBackupMemoryChunk()) {
            mem = MemRealloc(ptr, size);
        }

        if (!mem) {
            ReportAndExit("Failed to reallocate raw from backup pool");
        }
    }

    return mem;
}

void SafeAlloc::FreeRaw(nptr<void> ptr) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

    MemFree(ptr);
}

auto SafeAlloc::MallocAlignedRaw(size_t size, size_t alignment) noexcept -> nptr<void>
{
    FO_NO_STACK_TRACE_ENTRY();

    if (alignment == 0 || (alignment & (alignment - 1)) != 0) {
        ReportBadAlloc("Raw aligned malloc received invalid alignment", "byte", alignment, size);
        ReportAndExit("Raw aligned allocation alignment is invalid");
    }

    nptr<void> mem = MemAlignedMalloc(size, alignment);

    if (!mem && size != 0) {
        ReportBadAlloc("Raw aligned malloc failed", "byte", alignment, size);

        while (!mem && FreeBackupMemoryChunk()) {
            mem = MemAlignedMalloc(size, alignment);
        }

        if (!mem) {
            ReportAndExit("Failed to allocate raw aligned from backup pool");
        }
    }

    return mem;
}

void SafeAlloc::FreeAlignedRaw(nptr<void> ptr) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

    MemAlignedFree(ptr);
}

static auto MemMalloc(size_t size) noexcept -> nptr<void>
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_HAVE_RPMALLOC && FO_TRACY
    tracy::InitRpmalloc();
    void* p = tracy::rpmalloc(size);
    TracyAlloc(p, size);
    return p;
#elif FO_HAVE_RPMALLOC && !FO_TRACY
    return rpmalloc(size);
#else
    return malloc(size);
#endif
}

static auto MemCalloc(size_t num, size_t size) noexcept -> nptr<void>
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_HAVE_RPMALLOC && FO_TRACY
    tracy::InitRpmalloc();
    const auto result_size = num * size;
    if (num != 0 && size != 0 && result_size / num != size) {
        return nullptr; // Overflow
    }
    void* p = tracy::rpmalloc(result_size);
    if (p != nullptr) {
        MemFill(p, 0, result_size);
    }
    TracyAlloc(p, result_size);
    return p;
#elif FO_HAVE_RPMALLOC && !FO_TRACY
    return rpcalloc(num, size);
#else
    return calloc(num, size);
#endif
}

static auto MemRealloc(nptr<void> ptr, size_t size) noexcept -> nptr<void>
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_HAVE_RPMALLOC && FO_TRACY
    tracy::InitRpmalloc();
    TracyFree(ptr.get());
    void* p = tracy::rprealloc(ptr.get(), size);
    TracyAlloc(p, size);
    return p;
#elif FO_HAVE_RPMALLOC && !FO_TRACY
    return rprealloc(ptr.get(), size);
#else
    return realloc(ptr.get(), size);
#endif
}

static void MemFree(nptr<void> ptr) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_HAVE_RPMALLOC && FO_TRACY
    TracyFree(ptr.get());
    tracy::rpfree(ptr.get());
#elif FO_HAVE_RPMALLOC && !FO_TRACY
    rpfree(ptr.get());
#else
    free(ptr.get());
#endif
}

static auto MemAlignedMalloc(size_t size, size_t alignment) noexcept -> nptr<void>
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_HAVE_RPMALLOC && FO_TRACY
    tracy::InitRpmalloc();
    void* p = tracy::rpaligned_alloc(alignment, size);
    TracyAlloc(p, size);
    return p;
#elif FO_HAVE_RPMALLOC && !FO_TRACY
    return rpaligned_alloc(alignment, size);
#elif FO_WINDOWS
    return _aligned_malloc(size, alignment);
#else
    void* p = nullptr;
    // posix_memalign rejects alignments below sizeof(void*); over-aligning is always safe
    size_t effective_alignment = std::max(alignment, sizeof(void*));

    if (::posix_memalign(&p, effective_alignment, size) != 0) {
        return nullptr;
    }

    return p;
#endif
}

static void MemAlignedFree(nptr<void> ptr) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_HAVE_RPMALLOC && FO_TRACY
    TracyFree(ptr.get());
    tracy::rpfree(ptr.get());
#elif FO_HAVE_RPMALLOC && !FO_TRACY
    rpfree(ptr.get());
#elif FO_WINDOWS
    _aligned_free(ptr.get());
#else
    free(ptr.get());
#endif
}

extern auto AllocatorGetInUseBytes() noexcept -> size_t
{
    FO_NO_STACK_TRACE_ENTRY();

#if FO_HAVE_RPMALLOC && (FO_DEBUG || FO_TRACY)
#if FO_TRACY
    tracy::rpmalloc_global_statistics_t stats {};
    tracy::rpmalloc_global_statistics(&stats);

    size_t mapped = stats.mapped;
    size_t cached = stats.cached;

    if (mapped >= cached) {
        return mapped - cached;
    }
    return mapped;
#else
    rpmalloc_global_statistics_t stats {};
    ::rpmalloc_global_statistics(&stats);

    return stats.active;
#endif

#else
    return 0;
#endif
}

extern void InitBackupMemoryChunks()
{
    FO_STACK_TRACE_ENTRY();

    unique_arr_ptr<unique_arr_ptr<uint8_t>> new_chunks {new unique_arr_ptr<uint8_t>[BACKUP_MEMORY_CHUNKS]()};

    for (size_t i = 0; i < BACKUP_MEMORY_CHUNKS; i++) {
        new_chunks[i] = unique_arr_ptr<uint8_t> {new uint8_t[BACKUP_MEMORY_CHUNK_SIZE]()};
    }

    BackupMemoryChunks = std::move(new_chunks);
    BackupMemoryChunksCount.store(BACKUP_MEMORY_CHUNKS);
}

extern auto FreeBackupMemoryChunk() noexcept -> bool
{
    FO_NO_STACK_TRACE_ENTRY();

    while (true) {
        size_t cur_size = BackupMemoryChunksCount.load();

        if (cur_size == 0) {
            return false;
        }

        if (BackupMemoryChunksCount.compare_exchange_strong(cur_size, cur_size - 1)) {
            BackupMemoryChunks[cur_size - 1].reset();
            return true;
        }
    }
}

void SetBadAllocCallback(BadAllocCallback callback) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

    MemorySystem->Callback = std::move(callback);
}

extern void ReportBadAlloc(string_view message, string_view type_str, size_t count, size_t size) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

    BreakIntoDebugger();

    char itoa_buf[64] = {};

    WriteBaseLog("\nBAD ALLOC!\n\n");
    WriteBaseLog(message);
    WriteBaseLog("\n");
    WriteBaseLog("Type: ");
    WriteBaseLog(type_str);
    WriteBaseLog("\n");
    WriteBaseLog("Count: ");
    WriteBaseLog(ItoA(static_cast<int64_t>(count), itoa_buf, 10));
    WriteBaseLog("\n");
    WriteBaseLog("Size: ");
    WriteBaseLog(ItoA(static_cast<int64_t>(size), itoa_buf, 10));
    WriteBaseLog("\n\n");
    SafeWriteStackTrace(GetStackTrace());

    if (MemorySystem->Callback) {
        MemorySystem->Callback();
    }
}

extern void ReportAndExit(string_view message) noexcept
{
    FO_NO_STACK_TRACE_ENTRY();

    WriteBaseLog(message);

    ExitApp(false);
}

FO_END_NAMESPACE