llvm-project

1
//===--- extra/modularize/ModularizeUtilities.cpp -------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements a class for loading and validating a module map or
10
// header list by checking that all headers in the corresponding directories
11
// are accounted for.
12
//
13
//===----------------------------------------------------------------------===//
14

15
#include "clang/Basic/SourceManager.h"
16
#include "clang/Driver/Options.h"
17
#include "clang/Frontend/CompilerInstance.h"
18
#include "clang/Frontend/FrontendActions.h"
19
#include "CoverageChecker.h"
20
#include "llvm/ADT/SmallString.h"
21
#include "llvm/Support/FileUtilities.h"
22
#include "llvm/Support/MemoryBuffer.h"
23
#include "llvm/Support/Path.h"
24
#include "llvm/Support/raw_ostream.h"
25
#include "ModularizeUtilities.h"
26

27
using namespace clang;
28
using namespace llvm;
29
using namespace Modularize;
30

31
namespace {
32
// Subclass TargetOptions so we can construct it inline with
33
// the minimal option, the triple.
34
class ModuleMapTargetOptions : public clang::TargetOptions {
35
public:
36
  ModuleMapTargetOptions() { Triple = llvm::sys::getDefaultTargetTriple(); }
37
};
38
} // namespace
39

40
// ModularizeUtilities class implementation.
41

42
// Constructor.
43
ModularizeUtilities::ModularizeUtilities(std::vector<std::string> &InputPaths,
44
                                         llvm::StringRef Prefix,
45
                                         llvm::StringRef ProblemFilesListPath)
46
    : InputFilePaths(InputPaths), HeaderPrefix(Prefix),
47
      ProblemFilesPath(ProblemFilesListPath), HasModuleMap(false),
48
      MissingHeaderCount(0),
49
      // Init clang stuff needed for loading the module map and preprocessing.
50
      LangOpts(new LangOptions()), DiagIDs(new DiagnosticIDs()),
51
      DiagnosticOpts(new DiagnosticOptions()),
52
      DC(llvm::errs(), DiagnosticOpts.get()),
53
      Diagnostics(
54
          new DiagnosticsEngine(DiagIDs, DiagnosticOpts.get(), &DC, false)),
55
      TargetOpts(new ModuleMapTargetOptions()),
56
      Target(TargetInfo::CreateTargetInfo(*Diagnostics, TargetOpts)),
57
      FileMgr(new FileManager(FileSystemOpts)),
58
      SourceMgr(new SourceManager(*Diagnostics, *FileMgr, false)),
59
      HeaderInfo(new HeaderSearch(std::make_shared<HeaderSearchOptions>(),
60
                                  *SourceMgr, *Diagnostics, *LangOpts,
61
                                  Target.get())) {}
62

63
// Create instance of ModularizeUtilities, to simplify setting up
64
// subordinate objects.
65
ModularizeUtilities *ModularizeUtilities::createModularizeUtilities(
66
    std::vector<std::string> &InputPaths, llvm::StringRef Prefix,
67
    llvm::StringRef ProblemFilesListPath) {
68

69
  return new ModularizeUtilities(InputPaths, Prefix, ProblemFilesListPath);
70
}
71

72
// Load all header lists and dependencies.
73
std::error_code ModularizeUtilities::loadAllHeaderListsAndDependencies() {
74
  // For each input file.
75
  for (auto I = InputFilePaths.begin(), E = InputFilePaths.end(); I != E; ++I) {
76
    llvm::StringRef InputPath = *I;
77
    // If it's a module map.
78
    if (InputPath.ends_with(".modulemap")) {
79
      // Load the module map.
80
      if (std::error_code EC = loadModuleMap(InputPath))
81
        return EC;
82
    } else {
83
      // Else we assume it's a header list and load it.
84
      if (std::error_code EC = loadSingleHeaderListsAndDependencies(InputPath)) {
85
        errs() << "modularize: error: Unable to get header list '" << InputPath
86
          << "': " << EC.message() << '\n';
87
        return EC;
88
      }
89
    }
90
  }
91
  // If we have a problem files list.
92
  if (ProblemFilesPath.size() != 0) {
93
    // Load problem files list.
94
    if (std::error_code EC = loadProblemHeaderList(ProblemFilesPath)) {
95
      errs() << "modularize: error: Unable to get problem header list '" << ProblemFilesPath
96
        << "': " << EC.message() << '\n';
97
      return EC;
98
    }
99
  }
100
  return std::error_code();
101
}
102

103
// Do coverage checks.
104
// For each loaded module map, do header coverage check.
105
// Starting from the directory of the module.modulemap file,
106
// Find all header files, optionally looking only at files
107
// covered by the include path options, and compare against
108
// the headers referenced by the module.modulemap file.
109
// Display warnings for unaccounted-for header files.
110
// Returns 0 if there were no errors or warnings, 1 if there
111
// were warnings, 2 if any other problem, such as a bad
112
// module map path argument was specified.
113
std::error_code ModularizeUtilities::doCoverageCheck(
114
    std::vector<std::string> &IncludePaths,
115
    llvm::ArrayRef<std::string> CommandLine) {
116
  int ModuleMapCount = ModuleMaps.size();
117
  int ModuleMapIndex;
118
  std::error_code EC;
119
  for (ModuleMapIndex = 0; ModuleMapIndex < ModuleMapCount; ++ModuleMapIndex) {
120
    std::unique_ptr<clang::ModuleMap> &ModMap = ModuleMaps[ModuleMapIndex];
121
    auto Checker = CoverageChecker::createCoverageChecker(
122
        InputFilePaths[ModuleMapIndex], IncludePaths, CommandLine,
123
        ModMap.get());
124
    std::error_code LocalEC = Checker->doChecks();
125
    if (LocalEC.value() > 0)
126
      EC = LocalEC;
127
  }
128
  return EC;
129
}
130

131
// Load single header list and dependencies.
132
std::error_code ModularizeUtilities::loadSingleHeaderListsAndDependencies(
133
    llvm::StringRef InputPath) {
134

135
  // By default, use the path component of the list file name.
136
  SmallString<256> HeaderDirectory(InputPath);
137
  llvm::sys::path::remove_filename(HeaderDirectory);
138
  SmallString<256> CurrentDirectory;
139
  llvm::sys::fs::current_path(CurrentDirectory);
140

141
  // Get the prefix if we have one.
142
  if (HeaderPrefix.size() != 0)
143
    HeaderDirectory = HeaderPrefix;
144

145
  // Read the header list file into a buffer.
146
  ErrorOr<std::unique_ptr<MemoryBuffer>> listBuffer =
147
    MemoryBuffer::getFile(InputPath);
148
  if (std::error_code EC = listBuffer.getError())
149
    return EC;
150

151
  // Parse the header list into strings.
152
  SmallVector<StringRef, 32> Strings;
153
  listBuffer.get()->getBuffer().split(Strings, "\n", -1, false);
154

155
  // Collect the header file names from the string list.
156
  for (SmallVectorImpl<StringRef>::iterator I = Strings.begin(),
157
    E = Strings.end();
158
    I != E; ++I) {
159
    StringRef Line = I->trim();
160
    // Ignore comments and empty lines.
161
    if (Line.empty() || (Line[0] == '#'))
162
      continue;
163
    std::pair<StringRef, StringRef> TargetAndDependents = Line.split(':');
164
    SmallString<256> HeaderFileName;
165
    // Prepend header file name prefix if it's not absolute.
166
    if (llvm::sys::path::is_absolute(TargetAndDependents.first))
167
      llvm::sys::path::native(TargetAndDependents.first, HeaderFileName);
168
    else {
169
      if (HeaderDirectory.size() != 0)
170
        HeaderFileName = HeaderDirectory;
171
      else
172
        HeaderFileName = CurrentDirectory;
173
      llvm::sys::path::append(HeaderFileName, TargetAndDependents.first);
174
      llvm::sys::path::native(HeaderFileName);
175
    }
176
    // Handle optional dependencies.
177
    DependentsVector Dependents;
178
    SmallVector<StringRef, 4> DependentsList;
179
    TargetAndDependents.second.split(DependentsList, " ", -1, false);
180
    int Count = DependentsList.size();
181
    for (int Index = 0; Index < Count; ++Index) {
182
      SmallString<256> Dependent;
183
      if (llvm::sys::path::is_absolute(DependentsList[Index]))
184
        Dependent = DependentsList[Index];
185
      else {
186
        if (HeaderDirectory.size() != 0)
187
          Dependent = HeaderDirectory;
188
        else
189
          Dependent = CurrentDirectory;
190
        llvm::sys::path::append(Dependent, DependentsList[Index]);
191
      }
192
      llvm::sys::path::native(Dependent);
193
      Dependents.push_back(getCanonicalPath(Dependent.str()));
194
    }
195
    // Get canonical form.
196
    HeaderFileName = getCanonicalPath(HeaderFileName);
197
    // Save the resulting header file path and dependencies.
198
    HeaderFileNames.push_back(std::string(HeaderFileName));
199
    Dependencies[HeaderFileName.str()] = Dependents;
200
  }
201
  return std::error_code();
202
}
203

204
// Load problem header list.
205
std::error_code ModularizeUtilities::loadProblemHeaderList(
206
  llvm::StringRef InputPath) {
207

208
  // By default, use the path component of the list file name.
209
  SmallString<256> HeaderDirectory(InputPath);
210
  llvm::sys::path::remove_filename(HeaderDirectory);
211
  SmallString<256> CurrentDirectory;
212
  llvm::sys::fs::current_path(CurrentDirectory);
213

214
  // Get the prefix if we have one.
215
  if (HeaderPrefix.size() != 0)
216
    HeaderDirectory = HeaderPrefix;
217

218
  // Read the header list file into a buffer.
219
  ErrorOr<std::unique_ptr<MemoryBuffer>> listBuffer =
220
    MemoryBuffer::getFile(InputPath);
221
  if (std::error_code EC = listBuffer.getError())
222
    return EC;
223

224
  // Parse the header list into strings.
225
  SmallVector<StringRef, 32> Strings;
226
  listBuffer.get()->getBuffer().split(Strings, "\n", -1, false);
227

228
  // Collect the header file names from the string list.
229
  for (SmallVectorImpl<StringRef>::iterator I = Strings.begin(),
230
    E = Strings.end();
231
    I != E; ++I) {
232
    StringRef Line = I->trim();
233
    // Ignore comments and empty lines.
234
    if (Line.empty() || (Line[0] == '#'))
235
      continue;
236
    SmallString<256> HeaderFileName;
237
    // Prepend header file name prefix if it's not absolute.
238
    if (llvm::sys::path::is_absolute(Line))
239
      llvm::sys::path::native(Line, HeaderFileName);
240
    else {
241
      if (HeaderDirectory.size() != 0)
242
        HeaderFileName = HeaderDirectory;
243
      else
244
        HeaderFileName = CurrentDirectory;
245
      llvm::sys::path::append(HeaderFileName, Line);
246
      llvm::sys::path::native(HeaderFileName);
247
    }
248
    // Get canonical form.
249
    HeaderFileName = getCanonicalPath(HeaderFileName);
250
    // Save the resulting header file path.
251
    ProblemFileNames.push_back(std::string(HeaderFileName));
252
  }
253
  return std::error_code();
254
}
255

256
// Load single module map and extract header file list.
257
std::error_code ModularizeUtilities::loadModuleMap(
258
    llvm::StringRef InputPath) {
259
  // Get file entry for module.modulemap file.
260
  auto ModuleMapEntryOrErr = SourceMgr->getFileManager().getFileRef(InputPath);
261

262
  // return error if not found.
263
  if (!ModuleMapEntryOrErr) {
264
    llvm::errs() << "error: File \"" << InputPath << "\" not found.\n";
265
    return errorToErrorCode(ModuleMapEntryOrErr.takeError());
266
  }
267
  FileEntryRef ModuleMapEntry = *ModuleMapEntryOrErr;
268

269
  // Because the module map parser uses a ForwardingDiagnosticConsumer,
270
  // which doesn't forward the BeginSourceFile call, we do it explicitly here.
271
  DC.BeginSourceFile(*LangOpts, nullptr);
272

273
  // Figure out the home directory for the module map file.
274
  DirectoryEntryRef Dir = ModuleMapEntry.getDir();
275
  StringRef DirName(Dir.getName());
276
  if (llvm::sys::path::filename(DirName) == "Modules") {
277
    DirName = llvm::sys::path::parent_path(DirName);
278
    if (DirName.ends_with(".framework")) {
279
      auto FrameworkDirOrErr = FileMgr->getDirectoryRef(DirName);
280
      if (!FrameworkDirOrErr) {
281
        // This can happen if there's a race between the above check and the
282
        // removal of the directory.
283
        return errorToErrorCode(FrameworkDirOrErr.takeError());
284
      }
285
      Dir = *FrameworkDirOrErr;
286
    }
287
  }
288

289
  std::unique_ptr<ModuleMap> ModMap;
290
  ModMap.reset(new ModuleMap(*SourceMgr, *Diagnostics, *LangOpts,
291
    Target.get(), *HeaderInfo));
292

293
  // Parse module.modulemap file into module map.
294
  if (ModMap->parseModuleMapFile(ModuleMapEntry, false, Dir)) {
295
    return std::error_code(1, std::generic_category());
296
  }
297

298
  // Do matching end call.
299
  DC.EndSourceFile();
300

301
  // Reset missing header count.
302
  MissingHeaderCount = 0;
303

304
  if (!collectModuleMapHeaders(ModMap.get()))
305
    return std::error_code(1, std::generic_category());
306

307
  // Save module map.
308
  ModuleMaps.push_back(std::move(ModMap));
309

310
  // Indicate we are using module maps.
311
  HasModuleMap = true;
312

313
  // Return code of 1 for missing headers.
314
  if (MissingHeaderCount)
315
    return std::error_code(1, std::generic_category());
316

317
  return std::error_code();
318
}
319

320
// Collect module map headers.
321
// Walks the modules and collects referenced headers into
322
// HeaderFileNames.
323
bool ModularizeUtilities::collectModuleMapHeaders(clang::ModuleMap *ModMap) {
324
  SmallVector<std::pair<StringRef, const clang::Module *>, 0> Vec;
325
  for (auto &M : ModMap->modules())
326
    Vec.emplace_back(M.first(), M.second);
327
  llvm::sort(Vec, llvm::less_first());
328
  for (auto &I : Vec)
329
    if (!collectModuleHeaders(*I.second))
330
      return false;
331
  return true;
332
}
333

334
// Collect referenced headers from one module.
335
// Collects the headers referenced in the given module into
336
// HeaderFileNames.
337
bool ModularizeUtilities::collectModuleHeaders(const clang::Module &Mod) {
338

339
  // Ignore explicit modules because they often have dependencies
340
  // we can't know.
341
  if (Mod.IsExplicit)
342
    return true;
343

344
  // Treat headers in umbrella directory as dependencies.
345
  DependentsVector UmbrellaDependents;
346

347
  // Recursively do submodules.
348
  for (auto *Submodule : Mod.submodules())
349
    collectModuleHeaders(*Submodule);
350

351
  if (std::optional<clang::Module::Header> UmbrellaHeader =
352
          Mod.getUmbrellaHeaderAsWritten()) {
353
    std::string HeaderPath = getCanonicalPath(UmbrellaHeader->Entry.getName());
354
    // Collect umbrella header.
355
    HeaderFileNames.push_back(HeaderPath);
356

357
    // FUTURE: When needed, umbrella header header collection goes here.
358
  } else if (std::optional<clang::Module::DirectoryName> UmbrellaDir =
359
                 Mod.getUmbrellaDirAsWritten()) {
360
    // If there normal headers, assume these are umbrellas and skip collection.
361
    if (Mod.Headers->size() == 0) {
362
      // Collect headers in umbrella directory.
363
      if (!collectUmbrellaHeaders(UmbrellaDir->Entry.getName(),
364
                                  UmbrellaDependents))
365
        return false;
366
    }
367
  }
368

369
  // We ignore HK_Private, HK_Textual, HK_PrivateTextual, and HK_Excluded,
370
  // assuming they are marked as such either because of unsuitability for
371
  // modules or because they are meant to be included by another header,
372
  // and thus should be ignored by modularize.
373

374
  int NormalHeaderCount = Mod.Headers[clang::Module::HK_Normal].size();
375

376
  for (int Index = 0; Index < NormalHeaderCount; ++Index) {
377
    DependentsVector NormalDependents;
378
    // Collect normal header.
379
    const clang::Module::Header &Header(
380
      Mod.Headers[clang::Module::HK_Normal][Index]);
381
    std::string HeaderPath = getCanonicalPath(Header.Entry.getName());
382
    HeaderFileNames.push_back(HeaderPath);
383
  }
384

385
  int MissingCountThisModule = Mod.MissingHeaders.size();
386

387
  for (int Index = 0; Index < MissingCountThisModule; ++Index) {
388
    std::string MissingFile = Mod.MissingHeaders[Index].FileName;
389
    SourceLocation Loc = Mod.MissingHeaders[Index].FileNameLoc;
390
    errs() << Loc.printToString(*SourceMgr)
391
      << ": error : Header not found: " << MissingFile << "\n";
392
  }
393

394
  MissingHeaderCount += MissingCountThisModule;
395

396
  return true;
397
}
398

399
// Collect headers from an umbrella directory.
400
bool ModularizeUtilities::collectUmbrellaHeaders(StringRef UmbrellaDirName,
401
  DependentsVector &Dependents) {
402
  // Initialize directory name.
403
  SmallString<256> Directory(UmbrellaDirName);
404
  // Walk the directory.
405
  std::error_code EC;
406
  for (llvm::sys::fs::directory_iterator I(Directory.str(), EC), E; I != E;
407
    I.increment(EC)) {
408
    if (EC)
409
      return false;
410
    std::string File(I->path());
411
    llvm::ErrorOr<llvm::sys::fs::basic_file_status> Status = I->status();
412
    if (!Status)
413
      return false;
414
    llvm::sys::fs::file_type Type = Status->type();
415
    // If the file is a directory, ignore the name and recurse.
416
    if (Type == llvm::sys::fs::file_type::directory_file) {
417
      if (!collectUmbrellaHeaders(File, Dependents))
418
        return false;
419
      continue;
420
    }
421
    // If the file does not have a common header extension, ignore it.
422
    if (!isHeader(File))
423
      continue;
424
    // Save header name.
425
    std::string HeaderPath = getCanonicalPath(File);
426
    Dependents.push_back(HeaderPath);
427
  }
428
  return true;
429
}
430

431
// Replace .. embedded in path for purposes of having
432
// a canonical path.
433
static std::string replaceDotDot(StringRef Path) {
434
  SmallString<128> Buffer;
435
  llvm::sys::path::const_iterator B = llvm::sys::path::begin(Path),
436
    E = llvm::sys::path::end(Path);
437
  while (B != E) {
438
    if (*B == "..")
439
      llvm::sys::path::remove_filename(Buffer);
440
    else if (*B != ".")
441
      llvm::sys::path::append(Buffer, *B);
442
    ++B;
443
  }
444
  if (Path.ends_with("/") || Path.ends_with("\\"))
445
    Buffer.append(1, Path.back());
446
  return Buffer.c_str();
447
}
448

449
// Convert header path to canonical form.
450
// The canonical form is basically just use forward slashes, and remove "./".
451
// \param FilePath The file path, relative to the module map directory.
452
// \returns The file path in canonical form.
453
std::string ModularizeUtilities::getCanonicalPath(StringRef FilePath) {
454
  std::string Tmp(replaceDotDot(FilePath));
455
  std::replace(Tmp.begin(), Tmp.end(), '\\', '/');
456
  StringRef Tmp2(Tmp);
457
  if (Tmp2.starts_with("./"))
458
    Tmp = std::string(Tmp2.substr(2));
459
  return Tmp;
460
}
461

462
// Check for header file extension.
463
// If the file extension is .h, .inc, or missing, it's
464
// assumed to be a header.
465
// \param FileName The file name.  Must not be a directory.
466
// \returns true if it has a header extension or no extension.
467
bool ModularizeUtilities::isHeader(StringRef FileName) {
468
  StringRef Extension = llvm::sys::path::extension(FileName);
469
  if (Extension.size() == 0)
470
    return true;
471
  if (Extension.equals_insensitive(".h"))
472
    return true;
473
  if (Extension.equals_insensitive(".inc"))
474
    return true;
475
  return false;
476
}
477

478
// Get directory path component from file path.
479
// \returns the component of the given path, which will be
480
// relative if the given path is relative, absolute if the
481
// given path is absolute, or "." if the path has no leading
482
// path component.
483
std::string ModularizeUtilities::getDirectoryFromPath(StringRef Path) {
484
  SmallString<256> Directory(Path);
485
  sys::path::remove_filename(Directory);
486
  if (Directory.size() == 0)
487
    return ".";
488
  return std::string(Directory);
489
}
490

491
// Add unique problem file.
492
// Also standardizes the path.
493
void ModularizeUtilities::addUniqueProblemFile(std::string FilePath) {
494
  FilePath = getCanonicalPath(FilePath);
495
  // Don't add if already present.
496
  for(auto &TestFilePath : ProblemFileNames) {
497
    if (TestFilePath == FilePath)
498
      return;
499
  }
500
  ProblemFileNames.push_back(FilePath);
501
}
502

503
// Add file with no compile errors.
504
// Also standardizes the path.
505
void ModularizeUtilities::addNoCompileErrorsFile(std::string FilePath) {
506
  FilePath = getCanonicalPath(FilePath);
507
  GoodFileNames.push_back(FilePath);
508
}
509

510
// List problem files.
511
void ModularizeUtilities::displayProblemFiles() {
512
  errs() << "\nThese are the files with possible errors:\n\n";
513
  for (auto &ProblemFile : ProblemFileNames) {
514
    errs() << ProblemFile << "\n";
515
  }
516
}
517

518
// List files with no problems.
519
void ModularizeUtilities::displayGoodFiles() {
520
  errs() << "\nThese are the files with no detected errors:\n\n";
521
  for (auto &GoodFile : HeaderFileNames) {
522
    bool Good = true;
523
    for (auto &ProblemFile : ProblemFileNames) {
524
      if (ProblemFile == GoodFile) {
525
        Good = false;
526
        break;
527
      }
528
    }
529
    if (Good)
530
      errs() << GoodFile << "\n";
531
  }
532
}
533

534
// List files with problem files commented out.
535
void ModularizeUtilities::displayCombinedFiles() {
536
  errs() <<
537
    "\nThese are the combined files, with problem files preceded by #:\n\n";
538
  for (auto &File : HeaderFileNames) {
539
    bool Good = true;
540
    for (auto &ProblemFile : ProblemFileNames) {
541
      if (ProblemFile == File) {
542
        Good = false;
543
        break;
544
      }
545
    }
546
    errs() << (Good ? "" : "#") << File << "\n";
547
  }
548
}
549