llvm-project

1
//===-- Benchmark ---------------------------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8

9
#include "JSON.h"
10
#include "LibcBenchmark.h"
11
#include "LibcMemoryBenchmark.h"
12
#include "MemorySizeDistributions.h"
13
#include "llvm/Support/CommandLine.h"
14
#include "llvm/Support/ErrorHandling.h"
15
#include "llvm/Support/FileSystem.h"
16
#include "llvm/Support/JSON.h"
17
#include "llvm/Support/MathExtras.h"
18
#include "llvm/Support/MemoryBuffer.h"
19
#include "llvm/Support/raw_ostream.h"
20

21
#include <cstring>
22
#include <unistd.h>
23

24
namespace LIBC_NAMESPACE {
25

26
extern void *memcpy(void *__restrict, const void *__restrict, size_t);
27
extern void *memmove(void *, const void *, size_t);
28
extern void *memset(void *, int, size_t);
29
extern void bzero(void *, size_t);
30
extern int memcmp(const void *, const void *, size_t);
31
extern int bcmp(const void *, const void *, size_t);
32

33
} // namespace LIBC_NAMESPACE
34

35
namespace llvm {
36
namespace libc_benchmarks {
37

38
static cl::opt<std::string>
39
    StudyName("study-name", cl::desc("The name for this study"), cl::Required);
40

41
static cl::opt<std::string>
42
    SizeDistributionName("size-distribution-name",
43
                         cl::desc("The name of the distribution to use"));
44

45
static cl::opt<bool> SweepMode(
46
    "sweep-mode",
47
    cl::desc(
48
        "If set, benchmark all sizes from sweep-min-size to sweep-max-size"));
49

50
static cl::opt<uint32_t>
51
    SweepMinSize("sweep-min-size",
52
                 cl::desc("The minimum size to use in sweep-mode"),
53
                 cl::init(0));
54

55
static cl::opt<uint32_t>
56
    SweepMaxSize("sweep-max-size",
57
                 cl::desc("The maximum size to use in sweep-mode"),
58
                 cl::init(256));
59

60
static cl::opt<uint32_t>
61
    AlignedAccess("aligned-access",
62
                  cl::desc("The alignment to use when accessing the buffers\n"
63
                           "Default is unaligned\n"
64
                           "Use 0 to disable address randomization"),
65
                  cl::init(1));
66

67
static cl::opt<std::string> Output("output",
68
                                   cl::desc("Specify output filename"),
69
                                   cl::value_desc("filename"), cl::init("-"));
70

71
static cl::opt<uint32_t>
72
    NumTrials("num-trials", cl::desc("The number of benchmarks run to perform"),
73
              cl::init(1));
74

75
#if defined(LIBC_BENCHMARK_FUNCTION_MEMCPY)
76
#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCPY
77
using BenchmarkSetup = CopySetup;
78
#elif defined(LIBC_BENCHMARK_FUNCTION_MEMMOVE)
79
#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMMOVE
80
using BenchmarkSetup = MoveSetup;
81
#elif defined(LIBC_BENCHMARK_FUNCTION_MEMSET)
82
#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMSET
83
using BenchmarkSetup = SetSetup;
84
#elif defined(LIBC_BENCHMARK_FUNCTION_BZERO)
85
#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BZERO
86
using BenchmarkSetup = SetSetup;
87
#elif defined(LIBC_BENCHMARK_FUNCTION_MEMCMP)
88
#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCMP
89
using BenchmarkSetup = ComparisonSetup;
90
#elif defined(LIBC_BENCHMARK_FUNCTION_BCMP)
91
#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BCMP
92
using BenchmarkSetup = ComparisonSetup;
93
#else
94
#error "Missing LIBC_BENCHMARK_FUNCTION_XXX definition"
95
#endif
96

97
struct MemfunctionBenchmarkBase : public BenchmarkSetup {
98
  MemfunctionBenchmarkBase() : ReportProgress(isatty(fileno(stdout))) {}
99
  virtual ~MemfunctionBenchmarkBase() {}
100

101
  virtual Study run() = 0;
102

103
  CircularArrayRef<ParameterBatch::ParameterType>
104
  generateBatch(size_t Iterations) {
105
    randomize();
106
    return cycle(ArrayRef(Parameters), Iterations);
107
  }
108

109
protected:
110
  Study createStudy() {
111
    Study Study;
112
    // Setup study.
113
    Study.StudyName = StudyName;
114
    Runtime &RI = Study.Runtime;
115
    RI.Host = HostState::get();
116
    RI.BufferSize = BufferSize;
117
    RI.BatchParameterCount = BatchSize;
118

119
    BenchmarkOptions &BO = RI.BenchmarkOptions;
120
    BO.MinDuration = std::chrono::milliseconds(1);
121
    BO.MaxDuration = std::chrono::seconds(1);
122
    BO.MaxIterations = 10'000'000U;
123
    BO.MinSamples = 4;
124
    BO.MaxSamples = 1000;
125
    BO.Epsilon = 0.01; // 1%
126
    BO.ScalingFactor = 1.4;
127

128
    StudyConfiguration &SC = Study.Configuration;
129
    SC.NumTrials = NumTrials;
130
    SC.IsSweepMode = SweepMode;
131
    SC.AccessAlignment = MaybeAlign(AlignedAccess);
132
    SC.Function = LIBC_BENCHMARK_FUNCTION_NAME;
133
    return Study;
134
  }
135

136
  void runTrials(const BenchmarkOptions &Options,
137
                 std::vector<Duration> &Measurements) {
138
    for (size_t i = 0; i < NumTrials; ++i) {
139
      const BenchmarkResult Result = benchmark(
140
          Options, *this, [this](ParameterBatch::ParameterType Parameter) {
141
            return Call(Parameter, LIBC_BENCHMARK_FUNCTION);
142
          });
143
      Measurements.push_back(Result.BestGuess);
144
      reportProgress(Measurements);
145
    }
146
  }
147

148
  virtual void randomize() = 0;
149

150
private:
151
  bool ReportProgress;
152

153
  void reportProgress(const std::vector<Duration> &Measurements) {
154
    if (!ReportProgress)
155
      return;
156
    static size_t LastPercent = -1;
157
    const size_t TotalSteps = Measurements.capacity();
158
    const size_t Steps = Measurements.size();
159
    const size_t Percent = 100 * Steps / TotalSteps;
160
    if (Percent == LastPercent)
161
      return;
162
    LastPercent = Percent;
163
    size_t I = 0;
164
    errs() << '[';
165
    for (; I <= Percent; ++I)
166
      errs() << '#';
167
    for (; I <= 100; ++I)
168
      errs() << '_';
169
    errs() << "] " << Percent << '%' << '\r';
170
  }
171
};
172

173
struct MemfunctionBenchmarkSweep final : public MemfunctionBenchmarkBase {
174
  MemfunctionBenchmarkSweep()
175
      : OffsetSampler(MemfunctionBenchmarkBase::BufferSize, SweepMaxSize,
176
                      MaybeAlign(AlignedAccess)) {}
177

178
  virtual void randomize() override {
179
    for (auto &P : Parameters) {
180
      P.OffsetBytes = OffsetSampler(Gen);
181
      P.SizeBytes = CurrentSweepSize;
182
      checkValid(P);
183
    }
184
  }
185

186
  virtual Study run() override {
187
    Study Study = createStudy();
188
    Study.Configuration.SweepModeMaxSize = SweepMaxSize;
189
    BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
190
    BO.MinDuration = std::chrono::milliseconds(1);
191
    BO.InitialIterations = 100;
192
    auto &Measurements = Study.Measurements;
193
    Measurements.reserve(NumTrials * SweepMaxSize);
194
    for (size_t Size = SweepMinSize; Size <= SweepMaxSize; ++Size) {
195
      CurrentSweepSize = Size;
196
      runTrials(BO, Measurements);
197
    }
198
    return Study;
199
  }
200

201
private:
202
  size_t CurrentSweepSize = 0;
203
  OffsetDistribution OffsetSampler;
204
  std::mt19937_64 Gen;
205
};
206

207
struct MemfunctionBenchmarkDistribution final
208
    : public MemfunctionBenchmarkBase {
209
  MemfunctionBenchmarkDistribution(MemorySizeDistribution Distribution)
210
      : Distribution(Distribution), Probabilities(Distribution.Probabilities),
211
        SizeSampler(Probabilities.begin(), Probabilities.end()),
212
        OffsetSampler(MemfunctionBenchmarkBase::BufferSize,
213
                      Probabilities.size() - 1, MaybeAlign(AlignedAccess)) {}
214

215
  virtual void randomize() override {
216
    for (auto &P : Parameters) {
217
      P.OffsetBytes = OffsetSampler(Gen);
218
      P.SizeBytes = SizeSampler(Gen);
219
      checkValid(P);
220
    }
221
  }
222

223
  virtual Study run() override {
224
    Study Study = createStudy();
225
    Study.Configuration.SizeDistributionName = Distribution.Name.str();
226
    BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
227
    BO.MinDuration = std::chrono::milliseconds(10);
228
    BO.InitialIterations = BatchSize * 10;
229
    auto &Measurements = Study.Measurements;
230
    Measurements.reserve(NumTrials);
231
    runTrials(BO, Measurements);
232
    return Study;
233
  }
234

235
private:
236
  MemorySizeDistribution Distribution;
237
  ArrayRef<double> Probabilities;
238
  std::discrete_distribution<unsigned> SizeSampler;
239
  OffsetDistribution OffsetSampler;
240
  std::mt19937_64 Gen;
241
};
242

243
void writeStudy(const Study &S) {
244
  std::error_code EC;
245
  raw_fd_ostream FOS(Output, EC);
246
  if (EC)
247
    report_fatal_error(Twine("Could not open file: ")
248
                           .concat(EC.message())
249
                           .concat(", ")
250
                           .concat(Output));
251
  json::OStream JOS(FOS);
252
  serializeToJson(S, JOS);
253
  FOS << "\n";
254
}
255

256
void main() {
257
  checkRequirements();
258
  if (!isPowerOf2_32(AlignedAccess))
259
    report_fatal_error(AlignedAccess.ArgStr +
260
                       Twine(" must be a power of two or zero"));
261

262
  const bool HasDistributionName = !SizeDistributionName.empty();
263
  if (SweepMode && HasDistributionName)
264
    report_fatal_error("Select only one of `--" + Twine(SweepMode.ArgStr) +
265
                       "` or `--" + Twine(SizeDistributionName.ArgStr) + "`");
266

267
  std::unique_ptr<MemfunctionBenchmarkBase> Benchmark;
268
  if (SweepMode)
269
    Benchmark.reset(new MemfunctionBenchmarkSweep());
270
  else
271
    Benchmark.reset(new MemfunctionBenchmarkDistribution(getDistributionOrDie(
272
        BenchmarkSetup::getDistributions(), SizeDistributionName)));
273
  writeStudy(Benchmark->run());
274
}
275

276
} // namespace libc_benchmarks
277
} // namespace llvm
278

279
#ifndef NDEBUG
280
#error For reproducibility benchmarks should not be compiled in DEBUG mode.
281
#endif
282

283
int main(int argc, char **argv) {
284
  llvm::cl::ParseCommandLineOptions(argc, argv);
285
  llvm::libc_benchmarks::main();
286
  return EXIT_SUCCESS;
287
}
288