llvm-project

1
//===--- SystemIncludeExtractor.cpp ------------------------------*- C++-*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
// Some compiler drivers have implicit search mechanism for system headers.
9
// This compilation database implementation tries to extract that information by
10
// executing the driver in verbose mode. gcc-compatible drivers print something
11
// like:
12
// ....
13
// ....
14
// #include <...> search starts here:
15
//  /usr/lib/gcc/x86_64-linux-gnu/7/include
16
//  /usr/local/include
17
//  /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
18
//  /usr/include/x86_64-linux-gnu
19
//  /usr/include
20
// End of search list.
21
// ....
22
// ....
23
// This component parses that output and adds each path to command line args
24
// provided by Base, after prepending them with -isystem. Therefore current
25
// implementation would not work with a driver that is not gcc-compatible.
26
//
27
// First argument of the command line received from underlying compilation
28
// database is used as compiler driver path. Due to this arbitrary binary
29
// execution, this mechanism is not used by default and only executes binaries
30
// in the paths that are explicitly included by the user.
31

32
#include "CompileCommands.h"
33
#include "GlobalCompilationDatabase.h"
34
#include "support/Logger.h"
35
#include "support/Threading.h"
36
#include "support/Trace.h"
37
#include "clang/Basic/Diagnostic.h"
38
#include "clang/Basic/DiagnosticIDs.h"
39
#include "clang/Basic/DiagnosticOptions.h"
40
#include "clang/Basic/TargetInfo.h"
41
#include "clang/Basic/TargetOptions.h"
42
#include "clang/Driver/Types.h"
43
#include "clang/Tooling/CompilationDatabase.h"
44
#include "llvm/ADT/ArrayRef.h"
45
#include "llvm/ADT/DenseMap.h"
46
#include "llvm/ADT/Hashing.h"
47
#include "llvm/ADT/IntrusiveRefCntPtr.h"
48
#include "llvm/ADT/STLExtras.h"
49
#include "llvm/ADT/ScopeExit.h"
50
#include "llvm/ADT/SmallString.h"
51
#include "llvm/ADT/SmallVector.h"
52
#include "llvm/ADT/StringExtras.h"
53
#include "llvm/ADT/StringRef.h"
54
#include "llvm/Support/ErrorHandling.h"
55
#include "llvm/Support/FileSystem.h"
56
#include "llvm/Support/MemoryBuffer.h"
57
#include "llvm/Support/Path.h"
58
#include "llvm/Support/Program.h"
59
#include "llvm/Support/Regex.h"
60
#include "llvm/Support/ScopedPrinter.h"
61
#include "llvm/Support/raw_ostream.h"
62
#include <cassert>
63
#include <cstddef>
64
#include <iterator>
65
#include <memory>
66
#include <optional>
67
#include <string>
68
#include <tuple>
69
#include <utility>
70
#include <vector>
71

72
namespace clang::clangd {
73
namespace {
74

75
struct DriverInfo {
76
  std::vector<std::string> SystemIncludes;
77
  std::string Target;
78
};
79

80
struct DriverArgs {
81
  // Name of the driver program to execute or absolute path to it.
82
  std::string Driver;
83
  // Whether certain includes should be part of query.
84
  bool StandardIncludes = true;
85
  bool StandardCXXIncludes = true;
86
  // Language to use while querying.
87
  std::string Lang;
88
  std::string Sysroot;
89
  std::string ISysroot;
90
  std::string Target;
91
  std::string Stdlib;
92
  llvm::SmallVector<std::string> Specs;
93

94
  bool operator==(const DriverArgs &RHS) const {
95
    return std::tie(Driver, StandardIncludes, StandardCXXIncludes, Lang,
96
                    Sysroot, ISysroot, Target, Stdlib, Specs) ==
97
           std::tie(RHS.Driver, RHS.StandardIncludes, RHS.StandardCXXIncludes,
98
                    RHS.Lang, RHS.Sysroot, RHS.ISysroot, RHS.Target, RHS.Stdlib,
99
                    RHS.Specs);
100
  }
101

102
  DriverArgs(const tooling::CompileCommand &Cmd, llvm::StringRef File) {
103
    llvm::SmallString<128> Driver(Cmd.CommandLine.front());
104
    // Driver is a not a single executable name but instead a path (either
105
    // relative or absolute).
106
    if (llvm::any_of(Driver,
107
                     [](char C) { return llvm::sys::path::is_separator(C); })) {
108
      llvm::sys::fs::make_absolute(Cmd.Directory, Driver);
109
    }
110
    this->Driver = Driver.str().str();
111
    for (size_t I = 0, E = Cmd.CommandLine.size(); I < E; ++I) {
112
      llvm::StringRef Arg = Cmd.CommandLine[I];
113

114
      // Look for Language related flags.
115
      if (Arg.consume_front("-x")) {
116
        if (Arg.empty() && I + 1 < E)
117
          Lang = Cmd.CommandLine[I + 1];
118
        else
119
          Lang = Arg.str();
120
      }
121
      // Look for standard/builtin includes.
122
      else if (Arg == "-nostdinc" || Arg == "--no-standard-includes")
123
        StandardIncludes = false;
124
      else if (Arg == "-nostdinc++")
125
        StandardCXXIncludes = false;
126
      // Figure out sysroot
127
      else if (Arg.consume_front("--sysroot")) {
128
        if (Arg.consume_front("="))
129
          Sysroot = Arg.str();
130
        else if (Arg.empty() && I + 1 < E)
131
          Sysroot = Cmd.CommandLine[I + 1];
132
      } else if (Arg.consume_front("-isysroot")) {
133
        if (Arg.empty() && I + 1 < E)
134
          ISysroot = Cmd.CommandLine[I + 1];
135
        else
136
          ISysroot = Arg.str();
137
      } else if (Arg.consume_front("--target=")) {
138
        Target = Arg.str();
139
      } else if (Arg.consume_front("-target")) {
140
        if (Arg.empty() && I + 1 < E)
141
          Target = Cmd.CommandLine[I + 1];
142
      } else if (Arg.consume_front("--stdlib")) {
143
        if (Arg.consume_front("="))
144
          Stdlib = Arg.str();
145
        else if (Arg.empty() && I + 1 < E)
146
          Stdlib = Cmd.CommandLine[I + 1];
147
      } else if (Arg.consume_front("-stdlib=")) {
148
        Stdlib = Arg.str();
149
      } else if (Arg.starts_with("-specs=")) {
150
        // clang requires a single token like `-specs=file` or `--specs=file`,
151
        // but gcc will accept two tokens like `--specs file`. Since the
152
        // compilation database is presumably correct, we just forward the flags
153
        // as-is.
154
        Specs.push_back(Arg.str());
155
      } else if (Arg.starts_with("--specs=")) {
156
        Specs.push_back(Arg.str());
157
      } else if (Arg == "--specs" && I + 1 < E) {
158
        Specs.push_back(Arg.str());
159
        Specs.push_back(Cmd.CommandLine[I + 1]);
160
      }
161
    }
162

163
    // Downgrade objective-c++-header (used in clangd's fallback flags for .h
164
    // files) to c++-header, as some drivers may fail to run the extraction
165
    // command if it contains `-xobjective-c++-header` and objective-c++ support
166
    // is not installed.
167
    // In practice, we don't see different include paths for the two on
168
    // clang+mac, which is the most common objectve-c compiler.
169
    if (Lang == "objective-c++-header") {
170
      Lang = "c++-header";
171
    }
172

173
    // If language is not explicit in the flags, infer from the file.
174
    // This is important as we want to cache each language separately.
175
    if (Lang.empty()) {
176
      llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
177
      auto Type = driver::types::lookupTypeForExtension(Ext);
178
      if (Type == driver::types::TY_INVALID) {
179
        elog("System include extraction: invalid file type for {0}", Ext);
180
      } else {
181
        Lang = driver::types::getTypeName(Type);
182
      }
183
    }
184
  }
185
  llvm::SmallVector<llvm::StringRef> render() const {
186
    // FIXME: Don't treat lang specially?
187
    assert(!Lang.empty());
188
    llvm::SmallVector<llvm::StringRef> Args = {"-x", Lang};
189
    if (!StandardIncludes)
190
      Args.push_back("-nostdinc");
191
    if (!StandardCXXIncludes)
192
      Args.push_back("-nostdinc++");
193
    if (!Sysroot.empty())
194
      Args.append({"--sysroot", Sysroot});
195
    if (!ISysroot.empty())
196
      Args.append({"-isysroot", ISysroot});
197
    if (!Target.empty())
198
      Args.append({"-target", Target});
199
    if (!Stdlib.empty())
200
      Args.append({"--stdlib", Stdlib});
201

202
    for (llvm::StringRef Spec : Specs) {
203
      Args.push_back(Spec);
204
    }
205

206
    return Args;
207
  }
208

209
  static DriverArgs getEmpty() { return {}; }
210

211
private:
212
  DriverArgs() = default;
213
};
214
} // namespace
215
} // namespace clang::clangd
216
namespace llvm {
217
using DriverArgs = clang::clangd::DriverArgs;
218
template <> struct DenseMapInfo<DriverArgs> {
219
  static DriverArgs getEmptyKey() {
220
    auto Driver = DriverArgs::getEmpty();
221
    Driver.Driver = "EMPTY_KEY";
222
    return Driver;
223
  }
224
  static DriverArgs getTombstoneKey() {
225
    auto Driver = DriverArgs::getEmpty();
226
    Driver.Driver = "TOMBSTONE_KEY";
227
    return Driver;
228
  }
229
  static unsigned getHashValue(const DriverArgs &Val) {
230
    unsigned FixedFieldsHash = llvm::hash_value(std::tuple{
231
        Val.Driver,
232
        Val.StandardIncludes,
233
        Val.StandardCXXIncludes,
234
        Val.Lang,
235
        Val.Sysroot,
236
        Val.ISysroot,
237
        Val.Target,
238
        Val.Stdlib,
239
    });
240

241
    unsigned SpecsHash =
242
        llvm::hash_combine_range(Val.Specs.begin(), Val.Specs.end());
243

244
    return llvm::hash_combine(FixedFieldsHash, SpecsHash);
245
  }
246
  static bool isEqual(const DriverArgs &LHS, const DriverArgs &RHS) {
247
    return LHS == RHS;
248
  }
249
};
250
} // namespace llvm
251
namespace clang::clangd {
252
namespace {
253
bool isValidTarget(llvm::StringRef Triple) {
254
  std::shared_ptr<TargetOptions> TargetOpts(new TargetOptions);
255
  TargetOpts->Triple = Triple.str();
256
  DiagnosticsEngine Diags(new DiagnosticIDs, new DiagnosticOptions,
257
                          new IgnoringDiagConsumer);
258
  llvm::IntrusiveRefCntPtr<TargetInfo> Target =
259
      TargetInfo::CreateTargetInfo(Diags, TargetOpts);
260
  return bool(Target);
261
}
262

263
std::optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {
264
  DriverInfo Info;
265
  const char SIS[] = "#include <...> search starts here:";
266
  const char SIE[] = "End of search list.";
267
  const char TS[] = "Target: ";
268
  llvm::SmallVector<llvm::StringRef> Lines;
269
  Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
270

271
  enum {
272
    Initial,            // Initial state: searching for target or includes list.
273
    IncludesExtracting, // Includes extracting.
274
    Done                // Includes and target extraction done.
275
  } State = Initial;
276
  bool SeenIncludes = false;
277
  bool SeenTarget = false;
278
  for (auto *It = Lines.begin(); State != Done && It != Lines.end(); ++It) {
279
    auto Line = *It;
280
    switch (State) {
281
    case Initial:
282
      if (!SeenIncludes && Line.trim() == SIS) {
283
        SeenIncludes = true;
284
        State = IncludesExtracting;
285
      } else if (!SeenTarget && Line.trim().starts_with(TS)) {
286
        SeenTarget = true;
287
        llvm::StringRef TargetLine = Line.trim();
288
        TargetLine.consume_front(TS);
289
        // Only detect targets that clang understands
290
        if (!isValidTarget(TargetLine)) {
291
          elog("System include extraction: invalid target \"{0}\", ignoring",
292
               TargetLine);
293
        } else {
294
          Info.Target = TargetLine.str();
295
          vlog("System include extraction: target extracted: \"{0}\"",
296
               TargetLine);
297
        }
298
      }
299
      break;
300
    case IncludesExtracting:
301
      if (Line.trim() == SIE) {
302
        State = SeenTarget ? Done : Initial;
303
      } else {
304
        Info.SystemIncludes.push_back(Line.trim().str());
305
        vlog("System include extraction: adding {0}", Line);
306
      }
307
      break;
308
    default:
309
      llvm_unreachable("Impossible state of the driver output parser");
310
      break;
311
    }
312
  }
313
  if (!SeenIncludes) {
314
    elog("System include extraction: start marker not found: {0}", Output);
315
    return std::nullopt;
316
  }
317
  if (State == IncludesExtracting) {
318
    elog("System include extraction: end marker missing: {0}", Output);
319
    return std::nullopt;
320
  }
321
  return std::move(Info);
322
}
323

324
std::optional<std::string> run(llvm::ArrayRef<llvm::StringRef> Argv,
325
                               bool OutputIsStderr) {
326
  llvm::SmallString<128> OutputPath;
327
  if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
328
                                                   OutputPath)) {
329
    elog("System include extraction: failed to create temporary file with "
330
         "error {0}",
331
         EC.message());
332
    return std::nullopt;
333
  }
334
  auto CleanUp = llvm::make_scope_exit(
335
      [&OutputPath]() { llvm::sys::fs::remove(OutputPath); });
336

337
  std::optional<llvm::StringRef> Redirects[] = {{""}, {""}, {""}};
338
  Redirects[OutputIsStderr ? 2 : 1] = OutputPath.str();
339

340
  std::string ErrMsg;
341
  if (int RC =
342
          llvm::sys::ExecuteAndWait(Argv.front(), Argv, /*Env=*/std::nullopt,
343
                                    Redirects, /*SecondsToWait=*/0,
344
                                    /*MemoryLimit=*/0, &ErrMsg)) {
345
    elog("System include extraction: driver execution failed with return code: "
346
         "{0} - '{1}'. Args: [{2}]",
347
         llvm::to_string(RC), ErrMsg, printArgv(Argv));
348
    return std::nullopt;
349
  }
350

351
  auto BufOrError = llvm::MemoryBuffer::getFile(OutputPath);
352
  if (!BufOrError) {
353
    elog("System include extraction: failed to read {0} with error {1}",
354
         OutputPath, BufOrError.getError().message());
355
    return std::nullopt;
356
  }
357
  return BufOrError.get().get()->getBuffer().str();
358
}
359

360
std::optional<DriverInfo>
361
extractSystemIncludesAndTarget(const DriverArgs &InputArgs,
362
                               const llvm::Regex &QueryDriverRegex) {
363
  trace::Span Tracer("Extract system includes and target");
364

365
  std::string Driver = InputArgs.Driver;
366
  if (!llvm::sys::path::is_absolute(Driver)) {
367
    auto DriverProgram = llvm::sys::findProgramByName(Driver);
368
    if (DriverProgram) {
369
      vlog("System include extraction: driver {0} expanded to {1}", Driver,
370
           *DriverProgram);
371
      Driver = *DriverProgram;
372
    } else {
373
      elog("System include extraction: driver {0} not found in PATH", Driver);
374
      return std::nullopt;
375
    }
376
  }
377

378
  SPAN_ATTACH(Tracer, "driver", Driver);
379
  SPAN_ATTACH(Tracer, "lang", InputArgs.Lang);
380

381
  // If driver was "../foo" then having to allowlist "/path/a/../foo" rather
382
  // than "/path/foo" is absurd.
383
  // Allow either to match the allowlist, then proceed with "/path/a/../foo".
384
  // This was our historical behavior, and it *could* resolve to something else.
385
  llvm::SmallString<256> NoDots(Driver);
386
  llvm::sys::path::remove_dots(NoDots, /*remove_dot_dot=*/true);
387
  if (!QueryDriverRegex.match(Driver) && !QueryDriverRegex.match(NoDots)) {
388
    vlog("System include extraction: not allowed driver {0}", Driver);
389
    return std::nullopt;
390
  }
391

392
  llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-v"};
393
  Args.append(InputArgs.render());
394
  // Input needs to go after Lang flags.
395
  Args.push_back("-");
396
  auto Output = run(Args, /*OutputIsStderr=*/true);
397
  if (!Output)
398
    return std::nullopt;
399

400
  std::optional<DriverInfo> Info = parseDriverOutput(*Output);
401
  if (!Info)
402
    return std::nullopt;
403

404
  // The built-in headers are tightly coupled to parser builtins.
405
  // (These are clang's "resource dir", GCC's GCC_INCLUDE_DIR.)
406
  // We should keep using clangd's versions, so exclude the queried builtins.
407
  // They're not specially marked in the -v output, but we can get the path
408
  // with `$DRIVER -print-file-name=include`.
409
  if (auto BuiltinHeaders =
410
          run({Driver, "-print-file-name=include"}, /*OutputIsStderr=*/false)) {
411
    auto Path = llvm::StringRef(*BuiltinHeaders).trim();
412
    if (!Path.empty() && llvm::sys::path::is_absolute(Path)) {
413
      auto Size = Info->SystemIncludes.size();
414
      llvm::erase(Info->SystemIncludes, Path);
415
      vlog("System includes extractor: builtin headers {0} {1}", Path,
416
           (Info->SystemIncludes.size() != Size)
417
               ? "excluded"
418
               : "not found in driver's response");
419
    }
420
  }
421

422
  log("System includes extractor: successfully executed {0}\n\tgot includes: "
423
      "\"{1}\"\n\tgot target: \"{2}\"",
424
      Driver, llvm::join(Info->SystemIncludes, ", "), Info->Target);
425
  return Info;
426
}
427

428
tooling::CompileCommand &
429
addSystemIncludes(tooling::CompileCommand &Cmd,
430
                  llvm::ArrayRef<std::string> SystemIncludes) {
431
  std::vector<std::string> ToAppend;
432
  for (llvm::StringRef Include : SystemIncludes) {
433
    // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
434
    ToAppend.push_back("-isystem");
435
    ToAppend.push_back(Include.str());
436
  }
437
  if (!ToAppend.empty()) {
438
    // Just append when `--` isn't present.
439
    auto InsertAt = llvm::find(Cmd.CommandLine, "--");
440
    Cmd.CommandLine.insert(InsertAt, std::make_move_iterator(ToAppend.begin()),
441
                           std::make_move_iterator(ToAppend.end()));
442
  }
443
  return Cmd;
444
}
445

446
tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd,
447
                                   const std::string &Target) {
448
  if (!Target.empty()) {
449
    // We do not want to override existing target with extracted one.
450
    for (llvm::StringRef Arg : Cmd.CommandLine) {
451
      if (Arg == "-target" || Arg.starts_with("--target="))
452
        return Cmd;
453
    }
454
    // Just append when `--` isn't present.
455
    auto InsertAt = llvm::find(Cmd.CommandLine, "--");
456
    Cmd.CommandLine.insert(InsertAt, "--target=" + Target);
457
  }
458
  return Cmd;
459
}
460

461
/// Converts a glob containing only ** or * into a regex.
462
std::string convertGlobToRegex(llvm::StringRef Glob) {
463
  std::string RegText;
464
  llvm::raw_string_ostream RegStream(RegText);
465
  RegStream << '^';
466
  for (size_t I = 0, E = Glob.size(); I < E; ++I) {
467
    if (Glob[I] == '*') {
468
      if (I + 1 < E && Glob[I + 1] == '*') {
469
        // Double star, accept any sequence.
470
        RegStream << ".*";
471
        // Also skip the second star.
472
        ++I;
473
      } else {
474
        // Single star, accept any sequence without a slash.
475
        RegStream << "[^/]*";
476
      }
477
    } else if (llvm::sys::path::is_separator(Glob[I]) &&
478
               llvm::sys::path::is_separator('/') &&
479
               llvm::sys::path::is_separator('\\')) {
480
      RegStream << R"([/\\])"; // Accept either slash on windows.
481
    } else {
482
      RegStream << llvm::Regex::escape(Glob.substr(I, 1));
483
    }
484
  }
485
  RegStream << '$';
486
  RegStream.flush();
487
  return RegText;
488
}
489

490
/// Converts a glob containing only ** or * into a regex.
491
llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
492
  assert(!Globs.empty() && "Globs cannot be empty!");
493
  std::vector<std::string> RegTexts;
494
  RegTexts.reserve(Globs.size());
495
  for (llvm::StringRef Glob : Globs)
496
    RegTexts.push_back(convertGlobToRegex(Glob));
497

498
  // Tempting to pass IgnoreCase, but we don't know the FS sensitivity.
499
  llvm::Regex Reg(llvm::join(RegTexts, "|"));
500
  assert(Reg.isValid(RegTexts.front()) &&
501
         "Created an invalid regex from globs");
502
  return Reg;
503
}
504

505
/// Extracts system includes from a trusted driver by parsing the output of
506
/// include search path and appends them to the commands coming from underlying
507
/// compilation database.
508
class SystemIncludeExtractor {
509
public:
510
  SystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs)
511
      : QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)) {}
512

513
  void operator()(tooling::CompileCommand &Cmd, llvm::StringRef File) const {
514
    if (Cmd.CommandLine.empty())
515
      return;
516

517
    DriverArgs Args(Cmd, File);
518
    if (Args.Lang.empty())
519
      return;
520
    if (auto Info = QueriedDrivers.get(Args, [&] {
521
          return extractSystemIncludesAndTarget(Args, QueryDriverRegex);
522
        })) {
523
      setTarget(addSystemIncludes(Cmd, Info->SystemIncludes), Info->Target);
524
    }
525
  }
526

527
private:
528
  // Caches includes extracted from a driver. Key is driver:lang.
529
  Memoize<llvm::DenseMap<DriverArgs, std::optional<DriverInfo>>> QueriedDrivers;
530
  llvm::Regex QueryDriverRegex;
531
};
532
} // namespace
533

534
SystemIncludeExtractorFn
535
getSystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs) {
536
  if (QueryDriverGlobs.empty())
537
    return nullptr;
538
  return SystemIncludeExtractor(QueryDriverGlobs);
539
}
540

541
} // namespace clang::clangd
542