openllm

1
#!/usr/bin/env python3
2
import itertools, os, token, tokenize
3

4
TOKEN_WHITELIST = [token.OP, token.NAME, token.NUMBER, token.STRING]
5

6
_ignored = ['_version.py']
7

8
_dir_package = {'openllm-python': 'openllm', 'openllm-core': 'openllm_core', 'openllm-client': 'openllm_client'}
9

10

11
def run_cz(args):
12
  from tabulate import tabulate
13

14
  headers = ['Name', 'Lines', 'Tokens/Line']
15
  table = []
16
  package = _dir_package[args.dir]
17
  for path, _, files in os.walk(os.path.join(args.dir, 'src', package)):
18
    for name in files:
19
      if not name.endswith('.py') or name in _ignored:
20
        continue
21
      filepath = os.path.join(path, name)
22
      with tokenize.open(filepath) as file_:
23
        tokens = [t for t in tokenize.generate_tokens(file_.readline) if t.type in TOKEN_WHITELIST]
24
        token_count, line_count = len(tokens), len(set([t.start[0] for t in tokens]))
25
        table.append([
26
          filepath.replace(os.path.join(args.dir, 'src'), ''),
27
          line_count,
28
          token_count / line_count if line_count != 0 else 0,
29
        ])
30
  print(tabulate([headers, *sorted(table, key=lambda x: -x[1])], headers='firstrow', floatfmt='.1f') + '\n')
31
  print(
32
    tabulate(
33
      [
34
        (dir_name, sum([x[1] for x in group]))
35
        for dir_name, group in itertools.groupby(
36
          sorted([(x[0].rsplit('/', 1)[0], x[1]) for x in table]), key=lambda x: x[0]
37
        )
38
      ],
39
      headers=['Directory', 'LOC'],
40
      floatfmt='.1f',
41
    )
42
  )
43
  print(f'total line count for {package}: {sum([x[1] for x in table])}\n')
44
  return 0
45

46

47
if __name__ == '__main__':
48
  import argparse, importlib.util
49

50
  if importlib.util.find_spec('tabulate') is None:
51
    raise SystemExit('tabulate not installed. Install with `pip install tabulate`')
52

53
  parser = argparse.ArgumentParser()
54
  parser.add_argument(
55
    '--dir',
56
    choices=['openllm-python', 'openllm-core', 'openllm-client'],
57
    help='directory to check',
58
    default='openllm-python',
59
    required=False,
60
  )
61
  raise SystemExit(run_cz(parser.parse_args()))
62