CodeCompass

1
#include <memory>
2
#include <string>
3
#include <vector>
4
#include <iostream>
5
#include <fstream>
6

7
#include <boost/log/expressions.hpp>
8
#include <boost/log/expressions/attr.hpp>
9
#include <boost/log/attributes.hpp>
10
#include <boost/property_tree/ptree.hpp>
11
#include <boost/property_tree/json_parser.hpp>
12

13
#include <boost/program_options.hpp>
14
#include <boost/filesystem.hpp>
15

16
#include <util/dbutil.h>
17
#include <util/filesystem.h>
18
#include <util/logutil.h>
19
#include <util/odbtransaction.h>
20

21
#include <parser/parsercontext.h>
22
#include <parser/pluginhandler.h>
23
#include <parser/sourcemanager.h>
24

25
namespace po = boost::program_options;
26
namespace fs = boost::filesystem;
27
namespace trivial = boost::log::trivial;
28

29
po::options_description commandLineArguments()
30
{
31
  po::options_description desc("CodeCompass options");
32

33
  desc.add_options()
34
    ("help,h",
35
      "Prints this help message.")
36
    ("workspace,w", po::value<std::string>()->required(),
37
      "Path to a workspace directory. Project log files and plugin specific "
38
      "databases go under this directory.")
39
    ("name,n", po::value<std::string>()->required(),
40
      "Project name.")
41
    ("description", po::value<std::string>(),
42
      "Short description of the project.")
43
    ("force,f",
44
      "If the project already exists with the given name in the workspace, "
45
      "then by default CodeCompass skips all parsing actions. Force parsing "
46
      "removes all previous results from the workspace and from the database "
47
      "for the project.")
48
    ("list,l",
49
      "List available plugins. Plugins come from shared objects stored in the "
50
      "lib/parserplugin directory.")
51
    ("input,i", po::value<std::vector<std::string>>(),
52
      "The input of the parsers can be a compilation database (see: "
53
      "http://clang.llvm.org/docs/JSONCompilationDatabase.html) or a path of a "
54
      "directory under which the files will be consumed. Here you can list the "
55
      "compilation databases and the paths: -i /path/one -i /path/two -i "
56
      "compilation_database.json")
57
    ("database,d", po::value<std::string>()->required(),
58
      // TODO: Provide a full connection string example.
59
      "The parsers can use a relational database to store information. This "
60
      "database can be given by a connection string. Keep in mind that the "
61
      "database type depends on the CodeCompass executable. CodeCompass can be "
62
      "build to support PostgreSQL or SQLite. Connection string has the "
63
      "following format: 'pgsql:database=name;port=5432;user=user_name' or "
64
      "'sqlite:database=~/cc/mydatabase.sqlite'.")
65
    ("label", po::value<std::vector<std::string>>(),
66
      "The submodules of a large project can be labeled so it can be easier "
67
      "later to locate them. With this flag you can provide a label list in "
68
      "the following format: label1=/path/to/submodule1.")
69
    ("loglevel",
70
      po::value<trivial::severity_level>()->default_value(trivial::info),
71
      "Logging legel of the parser. Possible values are: debug, info, warning, "
72
      "error, critical.")
73
    ("logtarget", po::value<std::string>(),
74
      "This is the path to the folder where the logging output files will be written. "
75
      "If omitted, the output will be on the console only.")
76
    ("jobs,j", po::value<int>()->default_value(4),
77
      "Number of threads the parsers can use.")
78
    ("skip,s", po::value<std::vector<std::string>>(),
79
      "This is a list of parsers which will be omitted during the parsing "
80
      "process. The possible values are the plugin names which can be listed "
81
      "by --list flag.")
82
    ("dry-run",
83
     "Performs a dry on the incremental parsing maintenance, "
84
     "listing the detected changes, but without executing any "
85
     "further actions modifying the state of the database.")
86
    ("incremental-threshold", po::value<int>()->default_value(10),
87
      "This is a threshold percentage. If the total ratio of changed files "
88
      "is greater than this value, full parse is forced instead of incremental parsing.");
89

90
  return desc;
91
}
92

93
/**
94
 * This function checks the existence of the workspace and project directory
95
 * based on the given command line arguments.
96
 * @return Whether the project directory exists or not.
97
 */
98
bool checkProjectDir(const po::variables_map& vm_)
99
{
100
  const std::string projDir
101
    = vm_["workspace"].as<std::string>() + '/'
102
    + vm_["name"].as<std::string>();
103

104
  return fs::is_directory(projDir);
105
}
106

107
/**
108
 * This function prepares the workspace and project directory based on the given
109
 * command line arguments. If the project directory can't be created because of
110
 * permission issues then empty string returns which indicates the problem.
111
 * @return The path of the project directory.
112
 */
113
std::string prepareProjectDir(const po::variables_map& vm_)
114
{
115
  const std::string projDir
116
    = vm_["workspace"].as<std::string>() + '/'
117
    + vm_["name"].as<std::string>();
118

119
  boost::system::error_code ec;
120

121
  bool isNewProject = fs::create_directories(projDir, ec);
122

123
  if (ec)
124
  {
125
    LOG(error) << "Permission denied to create " + projDir;
126
    return std::string();
127
  }
128

129
  if (isNewProject)
130
    return projDir;
131

132
  if (vm_.count("force"))
133
  {
134
    fs::remove_all(projDir, ec);
135

136
    if (ec)
137
    {
138
      LOG(error) << "Permission denied to remove " + projDir;
139
      return std::string();
140
    }
141

142
    fs::create_directory(projDir, ec);
143

144
    if (ec)
145
    {
146
      LOG(error) << "Permission denied to create " + projDir;
147
      return std::string();
148
    }
149
  }
150

151
  return projDir;
152
}
153

154
/**
155
 * Lists the file statuses for added, modified and deleted files affected by
156
 * incremental parsing.
157
 * @param ctx_ Parser context.
158
 */
159
void incrementalList(cc::parser::ParserContext& ctx_)
160
{
161
  LOG(info) << "[Incremental parsing] Detected change list:";
162
  for (const auto& item : ctx_.fileStatus)
163
  {
164
    switch (item.second)
165
    {
166
      case cc::parser::IncrementalStatus::ADDED:
167
        LOG(info) << "ADDED file: " << item.first;
168
        break;
169
      case cc::parser::IncrementalStatus::MODIFIED:
170
        LOG(info) << "MODIFIED file: " << item.first;
171
        break;
172
      case cc::parser::IncrementalStatus::DELETED:
173
        LOG(info) << "DELETED file: " << item.first;
174
        break;
175
      case cc::parser::IncrementalStatus::ACTION_CHANGED:
176
        LOG(info) << "BUILD ACTION CHANGED file: " << item.first;
177
        break;
178
    }
179
  }
180
}
181

182
/**
183
 * Maintains and cleans up the file entries from the database as part of
184
 * incremental parsing.
185
 * @param ctx_ Parser context.
186
 */
187
void incrementalCleanup(cc::parser::ParserContext& ctx_)
188
{
189
  for (auto& item : ctx_.fileStatus)
190
  {
191
    cc::util::OdbTransaction {ctx_.db} ([&]
192
    {
193
      switch (item.second)
194
      {
195
        case cc::parser::IncrementalStatus::MODIFIED:
196
        case cc::parser::IncrementalStatus::DELETED:
197
        case cc::parser::IncrementalStatus::ACTION_CHANGED:
198
        {
199
          LOG(info) << "Database cleanup: " << item.first;
200

201
          // Fetch file from SourceManager by path
202
          cc::model::FilePtr delFile = ctx_.srcMgr.getFile(item.first);
203

204
          // Delete File and FileContent (only when no other File references it)
205
          ctx_.srcMgr.removeFile(*delFile);
206
          break;
207
        }
208
        case cc::parser::IncrementalStatus::ADDED:
209
          // Empty deliberately
210
          break;
211
      }
212
    });
213
  }
214
}
215

216
int main(int argc, char* argv[])
217
{
218
  std::string compassRoot = cc::util::binaryPathToInstallDir(argv[0]);
219

220
  const std::string PARSER_PLUGIN_DIR = compassRoot + "/lib/parserplugin";
221
  const std::string SQL_DIR = compassRoot + "/share/codecompass/sql";
222

223
  cc::parser::PluginHandler pHandler(PARSER_PLUGIN_DIR);
224

225
  cc::util::initConsoleLogger();
226

227
  //--- Process command line arguments ---//
228

229
  po::options_description desc = commandLineArguments();
230

231
  po::variables_map vm;
232
  po::store(po::command_line_parser(argc, argv)
233
    .options(desc).allow_unregistered().run(), vm);
234

235
  if (vm.count("logtarget"))
236
  {
237
    vm.at("logtarget").value() = cc::util::getLoggingBase( vm["logtarget"].as<std::string>()
238
                                                          , vm["name"].as<std::string>()
239
                                                          );
240
    if (!cc::util::initFileLogger(vm["logtarget"].as<std::string>() + "parser.log"))
241
    {
242
      vm.at("logtarget").value() = std::string();
243
    }
244
  }
245

246
  //--- Skip parser list ---//
247

248
  std::vector<std::string> skipParserList;
249
  if (vm.count("skip"))
250
    skipParserList = vm["skip"].as<std::vector<std::string>>();
251

252
  //--- Load parsers ---//
253

254
  pHandler.loadPlugins(skipParserList);
255

256
  //--- Add arguments of parsers ---//
257

258
  po::options_description pluginOptions = pHandler.getOptions();
259
  desc.add(pluginOptions);
260

261
  po::store(po::parse_command_line(argc, argv, desc), vm);
262

263
  if (argc < 2 || vm.count("help"))
264
  {
265
    std::cout << desc << std::endl;
266
    return 0;
267
  }
268

269
  if (vm.count("loglevel"))
270
  {
271
    trivial::severity_level loglevel
272
      = vm["loglevel"].as<trivial::severity_level>();
273
    boost::shared_ptr<boost::log::core> logger = boost::log::core::get();
274
    logger->set_filter(boost::log::expressions::attr<
275
      trivial::severity_level>("Severity") >= loglevel);
276
    logger->add_global_attribute("Severity",
277
      boost::log::attributes::mutable_constant<trivial::severity_level>(
278
        loglevel));
279
  }
280

281
  if (vm.count("list"))
282
  {
283
    std::cout << "Available plugins:" << std::endl;
284

285
    for (const std::string& pluginName : pHandler.getPluginNames())
286
      std::cout << " - " << pluginName << std::endl;
287

288
    return 0;
289
  }
290

291
  try
292
  {
293
    po::notify(vm);
294
  }
295
  catch (const po::error& e)
296
  {
297
    LOG(error) << "Error in command line arguments: " << e.what();
298
    return 1;
299
  }
300
  
301
  //--- Check database and project directory existence ---//
302
  
303
  bool isNewDb = cc::util::connectDatabase(
304
    vm["database"].as<std::string>(), false) == nullptr;
305
  bool isNewProject = !checkProjectDir(vm);
306

307
  if ((isNewProject ^ isNewDb) && !vm.count("force"))
308
  {
309
    LOG(error) << "Database and working directory existence are inconsistent. "
310
      "Use -f for reparsing!";
311
    return 1;
312
  }
313

314
  if (!isNewDb && !vm.count("force"))
315
  {
316
    LOG(info)
317
      << "Project already exists, incremental parsing in action"
318
      << (vm.count("dry-run") ? " (DRY RUN)" : "") << ".";
319
  }
320

321
  if (isNewDb && vm.count("dry-run"))
322
  {
323
    LOG(error) << "Dry-run can be only used with incremental parsing, "
324
      "no project found. Try turning --dry-run off.";
325
    return 1;
326
  }
327

328
  //--- Prepare workspace and project directory ---//
329
  
330
  std::string projDir = prepareProjectDir(vm);
331
  if (projDir.empty())
332
    return 1;
333

334
  //--- Create and init database ---//
335

336
  std::shared_ptr<odb::database> db = cc::util::connectDatabase(
337
    vm["database"].as<std::string>(), true);
338

339
  std::unordered_map<std::string, cc::parser::IncrementalStatus> fileStatus;
340

341
  if (!db)
342
  {
343
    LOG(error) << "Couldn't connect to database. Check the connection string. "
344
      "Connection string format: \"" +
345
      cc::util::getDbDriver() + ":<opt1>=<val1>;<opt2>=<val2>...\"";
346
    return 1;
347
  }
348

349
  if (vm.count("force"))
350
    cc::util::removeTables(db, SQL_DIR);
351

352
  if (vm.count("force") || isNewDb)
353
    cc::util::createTables(db, SQL_DIR);
354

355
  //--- Start parsers ---//
356

357
  /*
358
   * Workflow for incremental parsing:
359
   * 1. directly modified files are detected by ParserContext.
360
   * 2. all plugin parsers mark the indirectly modified files.
361
   * 3. all plugin parsers perform a cleanup operation.
362
   * 4. global tables are cleaned up by parser.cpp.
363
   * 5. all plugin parsers perform a parsing operation.
364
   *
365
   * In case of an initial or forced parsing, only step 5 is executed.
366
   */
367

368
  cc::parser::SourceManager srcMgr(db);
369
  cc::parser::ParserContext ctx(db, srcMgr, compassRoot, vm);
370
  pHandler.createPlugins(ctx);
371

372
  std::vector<std::string> pluginNames = pHandler.getLoadedPluginNames();
373
  for (const std::string& pluginName : pluginNames)
374
  {
375
    LOG(info) << "[" << pluginName << "] started to mark modified files!";
376
    pHandler.getParser(pluginName)->markModifiedFiles();
377
  }
378

379
  if (vm.count("dry-run"))
380
  {
381
    incrementalList(ctx);
382
    return 0;
383
  }
384

385
  if (ctx.fileStatus.size() >
386
    ctx.srcMgr.numberOfFiles() * vm["incremental-threshold"].as<int>() / 100.0)
387
  {
388
    LOG(info) << "The number of changed files exceeds the given incremental "
389
                 "threshold ratio, full parse will be forced.";
390
    vm.insert(std::make_pair("force", po::variable_value()));
391

392
    cc::util::removeTables(db, SQL_DIR);
393
    cc::util::createTables(db, SQL_DIR);
394

395
    srcMgr.reloadCache();
396
    ctx.fileStatus.clear();
397
  }
398

399
  if (!vm.count("force"))
400
  {
401
    for (const std::string& pluginName : pluginNames)
402
    {
403
      LOG(info) << "[" << pluginName << "] cleanup started!";
404
      if (!pHandler.getParser(pluginName)->cleanupDatabase())
405
      {
406
        LOG(error) << "[" << pluginName << "] cleanup failed!";
407
        return 2;
408
      }
409
    }
410

411
    incrementalCleanup(ctx);
412
  }
413

414
  // TODO: Handle errors returned by parse().
415
  for (const std::string& pluginName : pluginNames)
416
  {
417
    LOG(info) << "[" << pluginName << "] parse started!";
418
    pHandler.getParser(pluginName)->parse();
419
  }
420

421
  //--- Add indexes to the database ---//
422

423
  if (vm.count("force") || isNewDb)
424
    cc::util::createIndexes(db, SQL_DIR);
425

426
  //--- Create project config file ---//
427

428
  boost::property_tree::ptree pt;
429

430
  if (vm.count("label"))
431
  {
432
    boost::property_tree::ptree labels;
433

434
    for (const std::string& label : vm["label"].as<std::vector<std::string>>())
435
    {
436
      std::size_t pos = label.find('=');
437

438
      if (pos == std::string::npos)
439
        LOG(warning)
440
          << "Label doesn't contain '=' for separating label and the path: "
441
          << label;
442
      else
443
        labels.put(label.substr(0, pos), label.substr(pos + 1));
444
    }
445

446
    pt.add_child("labels", labels);
447
  }
448

449
  std::string database
450
    = cc::util::connStrComponent(vm["database"].as<std::string>(), "database");
451

452
  pt.put("database", vm["database"].as<std::string>());
453

454
  if (vm.count("description"))
455
    pt.put("description", vm["description"].as<std::string>());
456

457
  boost::property_tree::write_json(projDir + "/project_info.json", pt);
458

459
  // TODO: Print statistics.
460

461
  return 0;
462
}
463