llvm-project

1
//===--- Token.cpp - Tokens and token streams in the pseudoparser ---------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8

9
#include "clang-pseudo/Token.h"
10
#include "clang/Basic/LangOptions.h"
11
#include "llvm/ADT/StringExtras.h"
12
#include "llvm/Support/Format.h"
13
#include "llvm/Support/FormatVariadic.h"
14

15
namespace clang {
16
namespace pseudo {
17

18
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T) {
19
  OS << llvm::formatv("{0} {1}:{2} ", clang::tok::getTokenName(T.Kind), T.Line,
20
                      T.Indent);
21
  OS << '"';
22
  llvm::printEscapedString(T.text(), OS);
23
  OS << '"';
24
  if (T.Flags)
25
    OS << llvm::format(" flags=%x", T.Flags);
26
  return OS;
27
}
28

29
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const TokenStream &TS) {
30
  OS << "Index               Kind    Line  Text\n";
31
  for (const auto &T : TS.tokens()) {
32
    OS << llvm::format("%5d:  %16s %4d:%-2d  ", TS.index(T),
33
                       clang::tok::getTokenName(T.Kind), T.Line, T.Indent);
34
    OS << '"';
35
    llvm::printEscapedString(T.text(), OS);
36
    OS << '"';
37
    if (T.Flags)
38
      OS << llvm::format("  flags=%x", T.Flags);
39
    OS << '\n';
40
  }
41
  return OS;
42
}
43

44
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token::Range &R) {
45
  OS << llvm::formatv("[{0},{1})", R.Begin, R.End);
46
  return OS;
47
}
48

49
TokenStream::TokenStream(std::shared_ptr<void> Payload)
50
    : Payload(std::move(Payload)) {
51
  Storage.emplace_back();
52
  Storage.back().Kind = clang::tok::eof;
53
}
54

55
void TokenStream::finalize() {
56
  assert(!isFinalized());
57
  unsigned LastLine = Storage.back().Line;
58
  Storage.emplace_back();
59
  Storage.back().Kind = tok::eof;
60
  Storage.back().Line = LastLine + 1;
61

62
  Tokens = Storage;
63
  Tokens = Tokens.drop_front().drop_back();
64
}
65

66
bool TokenStream::isFinalized() const {
67
  assert(!Storage.empty() && Storage.front().Kind == tok::eof);
68
  if (Storage.size() == 1)
69
    return false;
70
  return Storage.back().Kind == tok::eof;
71
}
72

73
void TokenStream::print(llvm::raw_ostream &OS) const {
74
  bool FirstToken = true;
75
  unsigned LastLine = -1;
76
  StringRef LastText;
77
  for (const auto &T : tokens()) {
78
    StringRef Text = T.text();
79
    if (FirstToken) {
80
      FirstToken = false;
81
    } else if (T.Line == LastLine) {
82
      if (LastText.data() + LastText.size() != Text.data())
83
        OS << ' ';
84
    } else {
85
      OS << '\n';
86
      OS.indent(T.Indent);
87
    }
88
    OS << Text;
89
    LastLine = T.Line;
90
    LastText = Text;
91
  }
92
  if (!FirstToken)
93
    OS << '\n';
94
}
95

96
clang::LangOptions genericLangOpts(clang::Language Lang,
97
                                   clang::LangStandard::Kind Standard) {
98
  clang::LangOptions Opts;
99
  std::vector<std::string> UnusedIncludes;
100
  LangOptions::setLangDefaults(Opts, Lang, llvm::Triple(), UnusedIncludes,
101
                               Standard);
102

103
  // Some options are "on by default", but e.g. at the driver level.
104
  if (Opts.CPlusPlus)
105
    Opts.CXXOperatorNames = true;
106
  if (Opts.CPlusPlus20)
107
    Opts.Coroutines = true;
108

109
  // Some options are off by default, but define keywords we want to tolerate.
110
  if (Opts.CPlusPlus)
111
    Opts.MicrosoftExt = true;  // kw__try, kw__finally
112
  Opts.DeclSpecKeyword = true; // __declspec
113
  Opts.WChar = true;
114

115
  return Opts;
116
}
117

118
TokenStream stripComments(const TokenStream &Input) {
119
  TokenStream Out(Input.getPayload());
120
  for (const Token &T : Input.tokens()) {
121
    if (T.Kind == tok::comment)
122
      continue;
123
    Out.push(T);
124
  }
125
  Out.finalize();
126
  return Out;
127
}
128

129
} // namespace pseudo
130
} // namespace clang
131