llvm-project
483 строки · 13.7 Кб
1#!/usr/bin/env python
2
3import argparse4import sys5import os6
7from subprocess import call8
9SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))10PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects")11DEFAULT_LLVM_DIR = os.path.realpath(12os.path.join(SCRIPTS_DIR, os.path.pardir, os.path.pardir, os.path.pardir)13)
14
15
16def add(parser, args):17import SATestAdd18from ProjectMap import ProjectInfo19
20if args.source == "git" and (args.origin == "" or args.commit == ""):21parser.error("Please provide both --origin and --commit if source is 'git'")22
23if args.source != "git" and (args.origin != "" or args.commit != ""):24parser.error(25"Options --origin and --commit don't make sense when " "source is not 'git'"26)27
28project = ProjectInfo(29args.name[0], args.mode, args.source, args.origin, args.commit30)31
32SATestAdd.add_new_project(project)33
34
35def build(parser, args):36import SATestBuild37
38SATestBuild.VERBOSE = args.verbose39
40projects = get_projects(parser, args)41tester = SATestBuild.RegressionTester(42args.jobs,43projects,44args.override_compiler,45args.extra_analyzer_config,46args.extra_checkers,47args.regenerate,48args.strictness,49)50tests_passed = tester.test_all()51
52if not tests_passed:53sys.stderr.write("ERROR: Tests failed.\n")54sys.exit(42)55
56
57def compare(parser, args):58import CmpRuns59
60choices = [61CmpRuns.HistogramType.RELATIVE.value,62CmpRuns.HistogramType.LOG_RELATIVE.value,63CmpRuns.HistogramType.ABSOLUTE.value,64]65
66if args.histogram is not None and args.histogram not in choices:67parser.error(68"Incorrect histogram type, available choices are {}".format(choices)69)70
71dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)72dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)73
74CmpRuns.dump_scan_build_results_diff(75dir_old,76dir_new,77show_stats=args.show_stats,78stats_only=args.stats_only,79histogram=args.histogram,80verbose_log=args.verbose_log,81)82
83
84def update(parser, args):85import SATestUpdateDiffs86from ProjectMap import ProjectMap87
88project_map = ProjectMap()89for project in project_map.projects:90SATestUpdateDiffs.update_reference_results(project, args.git)91
92
93def benchmark(parser, args):94from SATestBenchmark import Benchmark95
96projects = get_projects(parser, args)97benchmark = Benchmark(projects, args.iterations, args.output)98benchmark.run()99
100
101def benchmark_compare(parser, args):102import SATestBenchmark103
104SATestBenchmark.compare(args.old, args.new, args.output)105
106
107def get_projects(parser, args):108from ProjectMap import ProjectMap, Size109
110project_map = ProjectMap()111projects = project_map.projects112
113def filter_projects(projects, predicate, force=False):114return [115project.with_fields(116enabled=(force or project.enabled) and predicate(project)117)118for project in projects119]120
121if args.projects:122projects_arg = args.projects.split(",")123available_projects = [project.name for project in projects]124
125# validate that given projects are present in the project map file126for manual_project in projects_arg:127if manual_project not in available_projects:128parser.error(129"Project '{project}' is not found in "130"the project map file. Available projects are "131"{all}.".format(project=manual_project, all=available_projects)132)133
134projects = filter_projects(135projects, lambda project: project.name in projects_arg, force=True136)137
138try:139max_size = Size.from_str(args.max_size)140except ValueError as e:141parser.error("{}".format(e))142
143projects = filter_projects(projects, lambda project: project.size <= max_size)144
145return projects146
147
148def docker(parser, args):149if len(args.rest) > 0:150if args.rest[0] != "--":151parser.error("REST arguments should start with '--'")152args.rest = args.rest[1:]153
154if args.build_image:155docker_build_image()156elif args.shell:157docker_shell(args)158else:159sys.exit(docker_run(args, " ".join(args.rest)))160
161
162def docker_build_image():163sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), shell=True))164
165
166def docker_shell(args):167try:168# First we need to start the docker container in a waiting mode,169# so it doesn't do anything, but most importantly keeps working170# while the shell session is in progress.171docker_run(args, "--wait", "--detach")172# Since the docker container is running, we can actually connect to it173call("docker exec -it satest bash", shell=True)174
175except KeyboardInterrupt:176pass177
178finally:179docker_cleanup()180
181
182def docker_run(args, command, docker_args=""):183try:184return call(185"docker run --rm --name satest "186"-v {llvm}:/llvm-project "187"-v {build}:/build "188"-v {clang}:/analyzer "189"-v {scripts}:/scripts "190"-v {projects}:/projects "191"{docker_args} "192"satest-image:latest {command}".format(193llvm=args.llvm_project_dir,194build=args.build_dir,195clang=args.clang_dir,196scripts=SCRIPTS_DIR,197projects=PROJECTS_DIR,198docker_args=docker_args,199command=command,200),201shell=True,202)203
204except KeyboardInterrupt:205docker_cleanup()206
207
208def docker_cleanup():209print("Please wait for docker to clean up")210call("docker stop satest", shell=True)211
212
213def main():214parser = argparse.ArgumentParser()215subparsers = parser.add_subparsers()216
217# add subcommand218add_parser = subparsers.add_parser(219"add", help="Add a new project for the analyzer testing."220)221# TODO: Add an option not to build.222# TODO: Set the path to the Repository directory.223add_parser.add_argument("name", nargs=1, help="Name of the new project")224add_parser.add_argument(225"--mode",226action="store",227default=1,228type=int,229choices=[0, 1, 2],230help="Build mode: 0 for single file project, "231"1 for scan_build, "232"2 for single file c++11 project",233)234add_parser.add_argument(235"--source",236action="store",237default="script",238choices=["script", "git", "zip"],239help="Source type of the new project: "240"'git' for getting from git "241"(please provide --origin and --commit), "242"'zip' for unpacking source from a zip file, "243"'script' for downloading source by running "244"a custom script",245)246add_parser.add_argument(247"--origin", action="store", default="", help="Origin link for a git repository"248)249add_parser.add_argument(250"--commit", action="store", default="", help="Git hash for a commit to checkout"251)252add_parser.set_defaults(func=add)253
254# build subcommand255build_parser = subparsers.add_parser(256"build",257help="Build projects from the project map and compare results with "258"the reference.",259)260build_parser.add_argument(261"--strictness",262dest="strictness",263type=int,264default=0,265help="0 to fail on runtime errors, 1 to fail "266"when the number of found bugs are different "267"from the reference, 2 to fail on any "268"difference from the reference. Default is 0.",269)270build_parser.add_argument(271"-r",272dest="regenerate",273action="store_true",274default=False,275help="Regenerate reference output.",276)277build_parser.add_argument(278"--override-compiler",279action="store_true",280default=False,281help="Call scan-build with " "--override-compiler option.",282)283build_parser.add_argument(284"-j",285"--jobs",286dest="jobs",287type=int,288default=0,289help="Number of projects to test concurrently",290)291build_parser.add_argument(292"--extra-analyzer-config",293dest="extra_analyzer_config",294type=str,295default="",296help="Arguments passed to to -analyzer-config",297)298build_parser.add_argument(299"--extra-checkers",300dest="extra_checkers",301type=str,302default="",303help="Extra checkers to enable",304)305build_parser.add_argument(306"--projects",307action="store",308default="",309help="Comma-separated list of projects to test",310)311build_parser.add_argument(312"--max-size",313action="store",314default=None,315help="Maximum size for the projects to test",316)317build_parser.add_argument("-v", "--verbose", action="count", default=0)318build_parser.set_defaults(func=build)319
320# compare subcommand321cmp_parser = subparsers.add_parser(322"compare",323help="Comparing two static analyzer runs in terms of "324"reported warnings and execution time statistics.",325)326cmp_parser.add_argument(327"--root-old",328dest="root_old",329help="Prefix to ignore on source files for " "OLD directory",330action="store",331type=str,332default="",333)334cmp_parser.add_argument(335"--root-new",336dest="root_new",337help="Prefix to ignore on source files for " "NEW directory",338action="store",339type=str,340default="",341)342cmp_parser.add_argument(343"--verbose-log",344dest="verbose_log",345help="Write additional information to LOG " "[default=None]",346action="store",347type=str,348default=None,349metavar="LOG",350)351cmp_parser.add_argument(352"--stats-only",353action="store_true",354dest="stats_only",355default=False,356help="Only show statistics on reports",357)358cmp_parser.add_argument(359"--show-stats",360action="store_true",361dest="show_stats",362default=False,363help="Show change in statistics",364)365cmp_parser.add_argument(366"--histogram",367action="store",368default=None,369help="Show histogram of paths differences. " "Requires matplotlib",370)371cmp_parser.add_argument("old", nargs=1, help="Directory with old results")372cmp_parser.add_argument("new", nargs=1, help="Directory with new results")373cmp_parser.set_defaults(func=compare)374
375# update subcommand376upd_parser = subparsers.add_parser(377"update",378help="Update static analyzer reference results based on the previous "379"run of SATest build. Assumes that SATest build was just run.",380)381upd_parser.add_argument(382"--git", action="store_true", help="Stage updated results using git."383)384upd_parser.set_defaults(func=update)385
386# docker subcommand387dock_parser = subparsers.add_parser(388"docker", help="Run regression system in the docker."389)390
391dock_parser.add_argument(392"--build-image",393action="store_true",394help="Build docker image for running tests.",395)396dock_parser.add_argument(397"--shell", action="store_true", help="Start a shell on docker."398)399dock_parser.add_argument(400"--llvm-project-dir",401action="store",402default=DEFAULT_LLVM_DIR,403help="Path to LLVM source code. Defaults "404"to the repo where this script is located. ",405)406dock_parser.add_argument(407"--build-dir",408action="store",409default="",410help="Path to a directory where docker should " "build LLVM code.",411)412dock_parser.add_argument(413"--clang-dir",414action="store",415default="",416help="Path to find/install LLVM installation.",417)418dock_parser.add_argument(419"rest",420nargs=argparse.REMAINDER,421default=[],422help="Additional args that will be forwarded " "to the docker's entrypoint.",423)424dock_parser.set_defaults(func=docker)425
426# benchmark subcommand427bench_parser = subparsers.add_parser(428"benchmark", help="Run benchmarks by building a set of projects multiple times."429)430
431bench_parser.add_argument(432"-i",433"--iterations",434action="store",435type=int,436default=20,437help="Number of iterations for building each " "project.",438)439bench_parser.add_argument(440"-o",441"--output",442action="store",443default="benchmark.csv",444help="Output csv file for the benchmark results",445)446bench_parser.add_argument(447"--projects",448action="store",449default="",450help="Comma-separated list of projects to test",451)452bench_parser.add_argument(453"--max-size",454action="store",455default=None,456help="Maximum size for the projects to test",457)458bench_parser.set_defaults(func=benchmark)459
460bench_subparsers = bench_parser.add_subparsers()461bench_compare_parser = bench_subparsers.add_parser(462"compare", help="Compare benchmark runs."463)464bench_compare_parser.add_argument(465"--old",466action="store",467required=True,468help="Benchmark reference results to " "compare agains.",469)470bench_compare_parser.add_argument(471"--new", action="store", required=True, help="New benchmark results to check."472)473bench_compare_parser.add_argument(474"-o", "--output", action="store", required=True, help="Output file for plots."475)476bench_compare_parser.set_defaults(func=benchmark_compare)477
478args = parser.parse_args()479args.func(parser, args)480
481
482if __name__ == "__main__":483main()484