cloud/blockstore/tools/testing/rdma-test/bootstrap.cpp - main - githubmirror/nbs

cloud/blockstore/tools/testing/rdma-test/bootstrap.cpp
242 строки7 KB
Daniil Komarevtsev
Add various parameters to the RDMA configs (#5911)
15 май 2026, 11:16
15 май 2026, 11:16d157982
#include "bootstrap.h"

#include "initiator.h"
#include "options.h"
#include "probes.h"
#include "storage.h"
#include "target.h"

#include <cloud/blockstore/libs/rdma/helper.h>

#include <cloud/storage/core/libs/common/scheduler.h>
#include <cloud/storage/core/libs/common/task_queue.h>
#include <cloud/storage/core/libs/common/thread_pool.h>
#include <cloud/storage/core/libs/common/timer.h>
#include <cloud/storage/core/libs/diagnostics/logging.h>
#include <cloud/storage/core/libs/diagnostics/monitoring.h>
#include <cloud/storage/core/libs/diagnostics/trace_processor_mon.h>
#include <cloud/storage/core/libs/diagnostics/trace_processor.h>
#include <cloud/storage/core/libs/diagnostics/trace_reader.h>
#include <cloud/storage/core/libs/rdma/iface/client.h>
#include <cloud/storage/core/libs/rdma/iface/probes.h>
#include <cloud/storage/core/libs/rdma/iface/server.h>

#include <library/cpp/lwtrace/mon/mon_lwtrace.h>

#include <util/generic/string.h>
#include <util/generic/vector.h>

namespace NCloud::NBlockStore {

////////////////////////////////////////////////////////////////////////////////

const TString TraceLoggerId = "st_trace_logger";

const TVector<std::tuple<TString, TString>> StartProbes = {
    {"RequestStarted", "BLOCKSTORE_TEST_PROVIDER"},
};

////////////////////////////////////////////////////////////////////////////////

TBootstrap::TBootstrap(TOptionsPtr options)
    : Options(std::move(options))
{}

void TBootstrap::Init()
{
    Timer = CreateWallClockTimer();
    Scheduler = CreateScheduler(Timer);

    InitLogging();
    InitTracing();

    Monitoring = CreateMonitoringServiceStub();

    if (Options->ThreadPool) {
        ThreadPool = CreateThreadPool(
            Options->TestMode == ETestMode::Target ? "SRV" : "CLI",
            Options->ThreadPool);
    } else {
        ThreadPool = CreateTaskQueueStub();
    }

    if (Options->TestMode == ETestMode::Target) {
        auto config =
            std::make_unique<NCloud::NStorage::NRdma::TServerConfig>();
        config->Backlog = Options->Backlog;
        config->QueueSize = Options->QueueSize;
        config->SendQueueSize = Options->SendQueueSize;
        config->RecvQueueSize = Options->RecvQueueSize;
        config->PollerThreads = Options->PollerThreads;
        config->WaitMode =
            NCloud::NStorage::NRdma::EWaitMode(Options->WaitMode);
        config->IpTypeOfService = Options->Tos;
        config->SourceInterface = Options->SourceInterface;
        config->VerbsQP = Options->VerbsQP;
        config->BufferPool = Options->BufferPool;
        config->QpRetryCount = Options->QpRetryCount;
        config->QpRnrRetryCount = Options->QpRnrRetryCount;
        config->QpTimeout = Options->QpTimeout;
        config->QpMinRnrTimer = Options->QpMinRnrTimer;

        Server = NCloud::NBlockStore::NRdma::CreateRdmaServer(
            Logging,
            Monitoring,
            std::move(config));
    } else if (Options->StorageKind == EStorageKind::Rdma) {
        auto config =
            std::make_unique<NCloud::NStorage::NRdma::TClientConfig>();
        config->QueueSize = Options->QueueSize;
        config->SendQueueSize = Options->SendQueueSize;
        config->RecvQueueSize = Options->RecvQueueSize;
        config->PollerThreads = Options->PollerThreads;
        config->WaitMode =
            NCloud::NStorage::NRdma::EWaitMode(Options->WaitMode);
        config->IpTypeOfService = Options->Tos;
        config->SourceInterface = Options->SourceInterface;
        config->VerbsQP = Options->VerbsQP;
        config->BufferPool = Options->BufferPool;
        config->ResolveTimeout =
            TDuration::MilliSeconds(Options->ResolveTimeout);
        config->FlushTimeout = TDuration::MilliSeconds(Options->FlushTimeout);
        config->QpRetryCount = Options->QpRetryCount;
        config->QpRnrRetryCount = Options->QpRnrRetryCount;
        config->QpTimeout = Options->QpTimeout;
        config->QpMinRnrTimer = Options->QpMinRnrTimer;

        Client = NCloud::NBlockStore::NRdma::CreateRdmaClient(
            Logging,
            Monitoring,
            std::move(config));
    }
}

void TBootstrap::InitLogging()
{
    TLogSettings settings;

    if (Options->VerboseLevel) {
        auto level = GetLogLevel(Options->VerboseLevel);
        Y_ENSURE(level, "unknown log level: " << Options->VerboseLevel.Quote());
        settings.FiltrationLevel = *level;
    }

    Logging = CreateLoggingService("console", settings);
}

void TBootstrap::InitTracing()
{
    auto& probes = NLwTraceMonPage::ProbeRegistry();
    probes.AddProbesList(LWTRACE_GET_PROBES(STORAGE_RDMA_PROVIDER));
    probes.AddProbesList(LWTRACE_GET_PROBES(BLOCKSTORE_TEST_PROVIDER));

    auto& traceManager = NLwTraceMonPage::TraceManager(false);

    TVector<ITraceReaderPtr> traceReaders;
    if (Options->TracePath) {
        traceManager.New(
            TraceLoggerId,
            ProbabilisticQuery(StartProbes, Options->TraceRate));

        traceReaders.emplace_back(SetupTraceReaderWithLog(
            TraceLoggerId,
            CreateLoggingService(Options->TracePath),
            "BLOCKSTORE_TRACE",
            "AllRequests"));
    }

    if (traceReaders) {
        TTraceProcessorConfig traceProcessorConfig;
        traceProcessorConfig.ComponentName = "BLOCKSTORE_TRACE";
        TraceProcessor = CreateTraceProcessorMon(
            Monitoring,
            CreateTraceProcessor(
                Timer,
                Scheduler,
                Logging,
                std::move(traceProcessorConfig),
                traceManager,
                std::move(traceReaders)));
    }
}

void TBootstrap::Start()
{
#define START(c) if (c) (c)->Start();

    START(Scheduler);
    START(Logging);
    START(Monitoring);
    START(ThreadPool);
    START(TraceProcessor);
    START(Server);
    START(Client);
    START(Storage);

#undef START
}

void TBootstrap::Stop()
{
#define STOP(c) if (c) (c)->Stop();

    STOP(Storage);
    STOP(Client);
    STOP(Server);
    STOP(TraceProcessor);
    STOP(ThreadPool);
    STOP(Monitoring);
    STOP(Logging);
    STOP(Scheduler);

#undef STOP
}

IRunnablePtr TBootstrap::CreateTest()
{
    switch (Options->StorageKind) {
    case EStorageKind::Null:
        Storage = CreateNullStorage();
        break;

    case EStorageKind::Memory:
        Storage = CreateMemoryStorage(
            Options->BlockSize,
            Options->BlocksCount);
        break;

    case EStorageKind::LocalAIO:
        Storage = CreateLocalAIOStorage(
            Options->StoragePath,
            Options->BlockSize,
            Options->BlocksCount);
        break;

    case EStorageKind::LocalURing:
        Storage = CreateLocalURingStorage(
            Options->StoragePath,
            Options->BlockSize,
            Options->BlocksCount);
        break;

    case EStorageKind::Rdma:
        Y_ABORT_UNLESS(Options->TestMode == ETestMode::Initiator);
        Storage = CreateRdmaStorage(
            Client,
            ThreadPool,
            Options->Host,
            Options->Port,
            TDuration::Seconds(Options->ConnectTimeout));
        break;
    }

    Storage->Start();

    if (Options->TestMode == ETestMode::Target) {
        return CreateTestTarget(Options, ThreadPool, Storage, Server);
    } else {
        return CreateTestInitiator(Options, Storage);
    }
}

}   // namespace NCloud::NBlockStore