llvm-project

1
//===-- YAMLSerialization.cpp ------------------------------------*- C++-*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// A YAML index file is a sequence of tagged entries.
10
// Each entry either encodes a Symbol or the list of references to a symbol
11
// (a "ref bundle").
12
//
13
//===----------------------------------------------------------------------===//
14

15
#include "Headers.h"
16
#include "index/Ref.h"
17
#include "index/Relation.h"
18
#include "index/Serialization.h"
19
#include "index/Symbol.h"
20
#include "index/SymbolLocation.h"
21
#include "index/SymbolOrigin.h"
22
#include "clang/Tooling/CompilationDatabase.h"
23
#include "llvm/ADT/StringRef.h"
24
#include "llvm/Support/Allocator.h"
25
#include "llvm/Support/StringSaver.h"
26
#include "llvm/Support/YAMLTraits.h"
27
#include "llvm/Support/raw_ostream.h"
28
#include <cstdint>
29
#include <optional>
30

31
namespace {
32
struct YIncludeHeaderWithReferences;
33
}
34

35
LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
36
LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)
37
LLVM_YAML_IS_SEQUENCE_VECTOR(YIncludeHeaderWithReferences)
38

39
namespace {
40
using RefBundle =
41
    std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
42
// This is a pale imitation of std::variant<Symbol, RefBundle, Relation>
43
struct VariantEntry {
44
  std::optional<clang::clangd::Symbol> Symbol;
45
  std::optional<RefBundle> Refs;
46
  std::optional<clang::clangd::Relation> Relation;
47
  std::optional<clang::clangd::IncludeGraphNode> Source;
48
  std::optional<clang::tooling::CompileCommand> Cmd;
49
};
50
// A class helps YAML to serialize the 32-bit encoded position (Line&Column),
51
// as YAMLIO can't directly map bitfields.
52
struct YPosition {
53
  uint32_t Line;
54
  uint32_t Column;
55
};
56
// A class helps YAML to serialize the IncludeHeaderWithReferences as YAMLIO
57
// can't directly map bitfields.
58
struct YIncludeHeaderWithReferences {
59
  llvm::StringRef IncludeHeader;
60
  uint32_t References;
61
  clang::clangd::Symbol::IncludeDirective SupportedDirectives;
62

63
  YIncludeHeaderWithReferences() = default;
64

65
  YIncludeHeaderWithReferences(
66
      llvm::StringRef IncludeHeader, uint32_t References,
67
      clang::clangd::Symbol::IncludeDirective SupportedDirectives)
68
      : IncludeHeader(IncludeHeader), References(References),
69
        SupportedDirectives(SupportedDirectives) {}
70
};
71

72
// avoid ODR violation of specialization for non-owned CompileCommand
73
struct CompileCommandYAML : clang::tooling::CompileCommand {};
74

75
} // namespace
76
namespace llvm {
77
namespace yaml {
78

79
using clang::clangd::FileDigest;
80
using clang::clangd::IncludeGraph;
81
using clang::clangd::IncludeGraphNode;
82
using clang::clangd::Ref;
83
using clang::clangd::RefKind;
84
using clang::clangd::Relation;
85
using clang::clangd::RelationKind;
86
using clang::clangd::Symbol;
87
using clang::clangd::SymbolID;
88
using clang::clangd::SymbolLocation;
89
using clang::index::SymbolInfo;
90
using clang::index::SymbolKind;
91
using clang::index::SymbolLanguage;
92
using clang::tooling::CompileCommand;
93

94
// Helper to (de)serialize the SymbolID. We serialize it as a hex string.
95
struct NormalizedSymbolID {
96
  NormalizedSymbolID(IO &) {}
97
  NormalizedSymbolID(IO &, const SymbolID &ID) {
98
    llvm::raw_string_ostream OS(HexString);
99
    OS << ID;
100
  }
101

102
  SymbolID denormalize(IO &I) {
103
    auto ID = SymbolID::fromStr(HexString);
104
    if (!ID) {
105
      I.setError(llvm::toString(ID.takeError()));
106
      return SymbolID();
107
    }
108
    return *ID;
109
  }
110

111
  std::string HexString;
112
};
113

114
struct NormalizedSymbolFlag {
115
  NormalizedSymbolFlag(IO &) {}
116
  NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) {
117
    Flag = static_cast<uint8_t>(F);
118
  }
119

120
  Symbol::SymbolFlag denormalize(IO &) {
121
    return static_cast<Symbol::SymbolFlag>(Flag);
122
  }
123

124
  uint8_t Flag = 0;
125
};
126

127
template <> struct MappingTraits<YPosition> {
128
  static void mapping(IO &IO, YPosition &Value) {
129
    IO.mapRequired("Line", Value.Line);
130
    IO.mapRequired("Column", Value.Column);
131
  }
132
};
133

134
struct NormalizedPosition {
135
  using Position = clang::clangd::SymbolLocation::Position;
136
  NormalizedPosition(IO &) {}
137
  NormalizedPosition(IO &, const Position &Pos) {
138
    P.Line = Pos.line();
139
    P.Column = Pos.column();
140
  }
141

142
  Position denormalize(IO &) {
143
    Position Pos;
144
    Pos.setLine(P.Line);
145
    Pos.setColumn(P.Column);
146
    return Pos;
147
  }
148
  YPosition P;
149
};
150

151
struct NormalizedFileURI {
152
  NormalizedFileURI(IO &) {}
153
  NormalizedFileURI(IO &, const char *FileURI) { URI = FileURI; }
154

155
  const char *denormalize(IO &IO) {
156
    assert(IO.getContext() &&
157
           "Expecting an UniqueStringSaver to allocate data");
158
    return static_cast<llvm::UniqueStringSaver *>(IO.getContext())
159
        ->save(URI)
160
        .data();
161
  }
162

163
  std::string URI;
164
};
165

166
template <> struct MappingTraits<SymbolLocation> {
167
  static void mapping(IO &IO, SymbolLocation &Value) {
168
    MappingNormalization<NormalizedFileURI, const char *> NFile(IO,
169
                                                                Value.FileURI);
170
    IO.mapRequired("FileURI", NFile->URI);
171
    MappingNormalization<NormalizedPosition, SymbolLocation::Position> NStart(
172
        IO, Value.Start);
173
    IO.mapRequired("Start", NStart->P);
174
    MappingNormalization<NormalizedPosition, SymbolLocation::Position> NEnd(
175
        IO, Value.End);
176
    IO.mapRequired("End", NEnd->P);
177
  }
178
};
179

180
template <> struct MappingTraits<SymbolInfo> {
181
  static void mapping(IO &IO, SymbolInfo &SymInfo) {
182
    // FIXME: expose other fields?
183
    IO.mapRequired("Kind", SymInfo.Kind);
184
    IO.mapRequired("Lang", SymInfo.Lang);
185
  }
186
};
187

188
template <> struct ScalarBitSetTraits<clang::clangd::Symbol::IncludeDirective> {
189
  static void bitset(IO &IO, clang::clangd::Symbol::IncludeDirective &Value) {
190
    IO.bitSetCase(Value, "Include", clang::clangd::Symbol::Include);
191
    IO.bitSetCase(Value, "Import", clang::clangd::Symbol::Import);
192
  }
193
};
194

195
template <> struct MappingTraits<YIncludeHeaderWithReferences> {
196
  static void mapping(IO &IO, YIncludeHeaderWithReferences &Inc) {
197
    IO.mapRequired("Header", Inc.IncludeHeader);
198
    IO.mapRequired("References", Inc.References);
199
    IO.mapOptional("Directives", Inc.SupportedDirectives,
200
                   clang::clangd::Symbol::Include);
201
  }
202
};
203

204
struct NormalizedIncludeHeaders {
205
  using IncludeHeader = clang::clangd::Symbol::IncludeHeaderWithReferences;
206
  NormalizedIncludeHeaders(IO &) {}
207
  NormalizedIncludeHeaders(
208
      IO &, const llvm::SmallVector<IncludeHeader, 1> &IncludeHeaders) {
209
    for (auto &I : IncludeHeaders) {
210
      Headers.emplace_back(I.IncludeHeader, I.References,
211
                           I.supportedDirectives());
212
    }
213
  }
214

215
  llvm::SmallVector<IncludeHeader, 1> denormalize(IO &) {
216
    llvm::SmallVector<IncludeHeader, 1> Result;
217
    for (auto &H : Headers)
218
      Result.emplace_back(H.IncludeHeader, H.References, H.SupportedDirectives);
219
    return Result;
220
  }
221
  llvm::SmallVector<YIncludeHeaderWithReferences, 1> Headers;
222
};
223

224
template <> struct MappingTraits<Symbol> {
225
  static void mapping(IO &IO, Symbol &Sym) {
226
    MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
227
    MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
228
        IO, Sym.Flags);
229
    MappingNormalization<
230
        NormalizedIncludeHeaders,
231
        llvm::SmallVector<Symbol::IncludeHeaderWithReferences, 1>>
232
        NIncludeHeaders(IO, Sym.IncludeHeaders);
233
    IO.mapRequired("ID", NSymbolID->HexString);
234
    IO.mapRequired("Name", Sym.Name);
235
    IO.mapRequired("Scope", Sym.Scope);
236
    IO.mapRequired("SymInfo", Sym.SymInfo);
237
    IO.mapOptional("CanonicalDeclaration", Sym.CanonicalDeclaration,
238
                   SymbolLocation());
239
    IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
240
    IO.mapOptional("References", Sym.References, 0u);
241
    IO.mapOptional("Flags", NSymbolFlag->Flag);
242
    IO.mapOptional("Signature", Sym.Signature);
243
    IO.mapOptional("TemplateSpecializationArgs",
244
                   Sym.TemplateSpecializationArgs);
245
    IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);
246
    IO.mapOptional("Documentation", Sym.Documentation);
247
    IO.mapOptional("ReturnType", Sym.ReturnType);
248
    IO.mapOptional("Type", Sym.Type);
249
    IO.mapOptional("IncludeHeaders", NIncludeHeaders->Headers);
250
  }
251
};
252

253
template <> struct ScalarEnumerationTraits<SymbolLanguage> {
254
  static void enumeration(IO &IO, SymbolLanguage &Value) {
255
    IO.enumCase(Value, "C", SymbolLanguage::C);
256
    IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
257
    IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
258
    IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
259
  }
260
};
261

262
template <> struct ScalarEnumerationTraits<SymbolKind> {
263
  static void enumeration(IO &IO, SymbolKind &Value) {
264
#define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
265

266
    DEFINE_ENUM(Unknown);
267
    DEFINE_ENUM(Function);
268
    DEFINE_ENUM(Module);
269
    DEFINE_ENUM(Namespace);
270
    DEFINE_ENUM(NamespaceAlias);
271
    DEFINE_ENUM(Macro);
272
    DEFINE_ENUM(Enum);
273
    DEFINE_ENUM(Struct);
274
    DEFINE_ENUM(Class);
275
    DEFINE_ENUM(Protocol);
276
    DEFINE_ENUM(Extension);
277
    DEFINE_ENUM(Union);
278
    DEFINE_ENUM(TypeAlias);
279
    DEFINE_ENUM(Function);
280
    DEFINE_ENUM(Variable);
281
    DEFINE_ENUM(Field);
282
    DEFINE_ENUM(EnumConstant);
283
    DEFINE_ENUM(InstanceMethod);
284
    DEFINE_ENUM(ClassMethod);
285
    DEFINE_ENUM(StaticMethod);
286
    DEFINE_ENUM(InstanceProperty);
287
    DEFINE_ENUM(ClassProperty);
288
    DEFINE_ENUM(StaticProperty);
289
    DEFINE_ENUM(Constructor);
290
    DEFINE_ENUM(Destructor);
291
    DEFINE_ENUM(ConversionFunction);
292
    DEFINE_ENUM(Parameter);
293
    DEFINE_ENUM(Using);
294

295
#undef DEFINE_ENUM
296
  }
297
};
298

299
template <> struct MappingTraits<RefBundle> {
300
  static void mapping(IO &IO, RefBundle &Refs) {
301
    MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO,
302
                                                                 Refs.first);
303
    IO.mapRequired("ID", NSymbolID->HexString);
304
    IO.mapRequired("References", Refs.second);
305
  }
306
};
307

308
struct NormalizedRefKind {
309
  NormalizedRefKind(IO &) {}
310
  NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); }
311

312
  RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); }
313

314
  uint8_t Kind = 0;
315
};
316

317
template <> struct MappingTraits<Ref> {
318
  static void mapping(IO &IO, Ref &R) {
319
    MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind);
320
    IO.mapRequired("Kind", NKind->Kind);
321
    IO.mapRequired("Location", R.Location);
322
  }
323
};
324

325
struct NormalizedSymbolRole {
326
  NormalizedSymbolRole(IO &) {}
327
  NormalizedSymbolRole(IO &IO, RelationKind R) {
328
    Kind = static_cast<uint8_t>(R);
329
  }
330

331
  RelationKind denormalize(IO &IO) { return static_cast<RelationKind>(Kind); }
332

333
  uint8_t Kind = 0;
334
};
335

336
template <> struct MappingTraits<SymbolID> {
337
  static void mapping(IO &IO, SymbolID &ID) {
338
    MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, ID);
339
    IO.mapRequired("ID", NSymbolID->HexString);
340
  }
341
};
342

343
template <> struct MappingTraits<Relation> {
344
  static void mapping(IO &IO, Relation &Relation) {
345
    MappingNormalization<NormalizedSymbolRole, RelationKind> NRole(
346
        IO, Relation.Predicate);
347
    IO.mapRequired("Subject", Relation.Subject);
348
    IO.mapRequired("Predicate", NRole->Kind);
349
    IO.mapRequired("Object", Relation.Object);
350
  }
351
};
352

353
struct NormalizedSourceFlag {
354
  NormalizedSourceFlag(IO &) {}
355
  NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) {
356
    Flag = static_cast<uint8_t>(O);
357
  }
358

359
  IncludeGraphNode::SourceFlag denormalize(IO &) {
360
    return static_cast<IncludeGraphNode::SourceFlag>(Flag);
361
  }
362

363
  uint8_t Flag = 0;
364
};
365

366
struct NormalizedFileDigest {
367
  NormalizedFileDigest(IO &) {}
368
  NormalizedFileDigest(IO &, const FileDigest &Digest) {
369
    HexString = llvm::toHex(Digest);
370
  }
371

372
  FileDigest denormalize(IO &I) {
373
    FileDigest Digest;
374
    if (HexString.size() == Digest.size() * 2 &&
375
        llvm::all_of(HexString, llvm::isHexDigit)) {
376
      memcpy(Digest.data(), llvm::fromHex(HexString).data(), Digest.size());
377
    } else {
378
      I.setError(std::string("Bad hex file digest: ") + HexString);
379
    }
380
    return Digest;
381
  }
382

383
  std::string HexString;
384
};
385

386
template <> struct MappingTraits<IncludeGraphNode> {
387
  static void mapping(IO &IO, IncludeGraphNode &Node) {
388
    IO.mapRequired("URI", Node.URI);
389
    MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag>
390
        NSourceFlag(IO, Node.Flags);
391
    IO.mapRequired("Flags", NSourceFlag->Flag);
392
    MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO,
393
                                                                   Node.Digest);
394
    IO.mapRequired("Digest", NDigest->HexString);
395
    IO.mapRequired("DirectIncludes", Node.DirectIncludes);
396
  }
397
};
398

399
template <> struct MappingTraits<CompileCommandYAML> {
400
  static void mapping(IO &IO, CompileCommandYAML &Cmd) {
401
    IO.mapRequired("Directory", Cmd.Directory);
402
    IO.mapRequired("CommandLine", Cmd.CommandLine);
403
  }
404
};
405

406
template <> struct MappingTraits<VariantEntry> {
407
  static void mapping(IO &IO, VariantEntry &Variant) {
408
    if (IO.mapTag("!Symbol", Variant.Symbol.has_value())) {
409
      if (!IO.outputting())
410
        Variant.Symbol.emplace();
411
      MappingTraits<Symbol>::mapping(IO, *Variant.Symbol);
412
    } else if (IO.mapTag("!Refs", Variant.Refs.has_value())) {
413
      if (!IO.outputting())
414
        Variant.Refs.emplace();
415
      MappingTraits<RefBundle>::mapping(IO, *Variant.Refs);
416
    } else if (IO.mapTag("!Relations", Variant.Relation.has_value())) {
417
      if (!IO.outputting())
418
        Variant.Relation.emplace();
419
      MappingTraits<Relation>::mapping(IO, *Variant.Relation);
420
    } else if (IO.mapTag("!Source", Variant.Source.has_value())) {
421
      if (!IO.outputting())
422
        Variant.Source.emplace();
423
      MappingTraits<IncludeGraphNode>::mapping(IO, *Variant.Source);
424
    } else if (IO.mapTag("!Cmd", Variant.Cmd.has_value())) {
425
      if (!IO.outputting())
426
        Variant.Cmd.emplace();
427
      MappingTraits<CompileCommandYAML>::mapping(
428
          IO, static_cast<CompileCommandYAML &>(*Variant.Cmd));
429
    }
430
  }
431
};
432

433
} // namespace yaml
434
} // namespace llvm
435

436
namespace clang {
437
namespace clangd {
438

439
void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) {
440
  llvm::yaml::Output Yout(OS);
441
  for (const auto &Sym : *O.Symbols) {
442
    VariantEntry Entry;
443
    Entry.Symbol = Sym;
444
    Yout << Entry;
445
  }
446
  if (O.Refs)
447
    for (auto &Sym : *O.Refs) {
448
      VariantEntry Entry;
449
      Entry.Refs = Sym;
450
      Yout << Entry;
451
    }
452
  if (O.Relations)
453
    for (auto &R : *O.Relations) {
454
      VariantEntry Entry;
455
      Entry.Relation = R;
456
      Yout << Entry;
457
    }
458
  if (O.Sources) {
459
    for (const auto &Source : *O.Sources) {
460
      VariantEntry Entry;
461
      Entry.Source = Source.getValue();
462
      Yout << Entry;
463
    }
464
  }
465
  if (O.Cmd) {
466
    VariantEntry Entry;
467
    Entry.Cmd = *O.Cmd;
468
    Yout << Entry;
469
  }
470
}
471

472
llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data,
473
                                     SymbolOrigin Origin) {
474
  SymbolSlab::Builder Symbols;
475
  RefSlab::Builder Refs;
476
  RelationSlab::Builder Relations;
477
  llvm::BumpPtrAllocator
478
      Arena; // store the underlying data of Position::FileURI.
479
  llvm::UniqueStringSaver Strings(Arena);
480
  llvm::yaml::Input Yin(Data, &Strings);
481
  IncludeGraph Sources;
482
  std::optional<tooling::CompileCommand> Cmd;
483
  while (Yin.setCurrentDocument()) {
484
    llvm::yaml::EmptyContext Ctx;
485
    VariantEntry Variant;
486
    yamlize(Yin, Variant, true, Ctx);
487
    if (Yin.error())
488
      return llvm::errorCodeToError(Yin.error());
489

490
    if (Variant.Symbol) {
491
      Variant.Symbol->Origin = Origin;
492
      Symbols.insert(*Variant.Symbol);
493
    }
494
    if (Variant.Refs)
495
      for (const auto &Ref : Variant.Refs->second)
496
        Refs.insert(Variant.Refs->first, Ref);
497
    if (Variant.Relation)
498
      Relations.insert(*Variant.Relation);
499
    if (Variant.Source) {
500
      auto &IGN = *Variant.Source;
501
      auto Entry = Sources.try_emplace(IGN.URI).first;
502
      Entry->getValue() = std::move(IGN);
503
      // Fixup refs to refer to map keys which will live on
504
      Entry->getValue().URI = Entry->getKey();
505
      for (auto &Include : Entry->getValue().DirectIncludes)
506
        Include = Sources.try_emplace(Include).first->getKey();
507
    }
508
    if (Variant.Cmd)
509
      Cmd = *Variant.Cmd;
510
    Yin.nextDocument();
511
  }
512

513
  IndexFileIn Result;
514
  Result.Symbols.emplace(std::move(Symbols).build());
515
  Result.Refs.emplace(std::move(Refs).build());
516
  Result.Relations.emplace(std::move(Relations).build());
517
  if (Sources.size())
518
    Result.Sources = std::move(Sources);
519
  Result.Cmd = std::move(Cmd);
520
  return std::move(Result);
521
}
522

523
std::string toYAML(const Symbol &S) {
524
  std::string Buf;
525
  {
526
    llvm::raw_string_ostream OS(Buf);
527
    llvm::yaml::Output Yout(OS);
528
    Symbol Sym = S; // copy: Yout<< requires mutability.
529
    Yout << Sym;
530
  }
531
  return Buf;
532
}
533

534
std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) {
535
  RefBundle Refs = {Data.first, Data.second};
536
  std::string Buf;
537
  {
538
    llvm::raw_string_ostream OS(Buf);
539
    llvm::yaml::Output Yout(OS);
540
    Yout << Refs;
541
  }
542
  return Buf;
543
}
544

545
std::string toYAML(const Relation &R) {
546
  std::string Buf;
547
  {
548
    llvm::raw_string_ostream OS(Buf);
549
    llvm::yaml::Output Yout(OS);
550
    Relation Rel = R; // copy: Yout<< requires mutability.
551
    Yout << Rel;
552
  }
553
  return Buf;
554
}
555

556
std::string toYAML(const Ref &R) {
557
  std::string Buf;
558
  {
559
    llvm::raw_string_ostream OS(Buf);
560
    llvm::yaml::Output Yout(OS);
561
    Ref Reference = R; // copy: Yout<< requires mutability.
562
    Yout << Reference;
563
  }
564
  return Buf;
565
}
566

567
} // namespace clangd
568
} // namespace clang
569