llvm-project
987 строк · 31.7 Кб
1#!/usr/bin/env python
2
3"""
4Static Analyzer qualification infrastructure.
5
6The goal is to test the analyzer against different projects,
7check for failures, compare results, and measure performance.
8
9Repository Directory will contain sources of the projects as well as the
10information on how to build them and the expected output.
11Repository Directory structure:
12- ProjectMap file
13- Historical Performance Data
14- Project Dir1
15- ReferenceOutput
16- Project Dir2
17- ReferenceOutput
18..
19Note that the build tree must be inside the project dir.
20
21To test the build of the analyzer one would:
22- Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that
23the build directory does not pollute the repository to min network
24traffic).
25- Build all projects, until error. Produce logs to report errors.
26- Compare results.
27
28The files which should be kept around for failure investigations:
29RepositoryCopy/Project DirI/ScanBuildResults
30RepositoryCopy/Project DirI/run_static_analyzer.log
31
32Assumptions (TODO: shouldn't need to assume these.):
33The script is being run from the Repository Directory.
34The compiler for scan-build and scan-build are in the PATH.
35export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH
36
37For more logging, set the env variables:
38zaks:TI zaks$ export CCC_ANALYZER_LOG=1
39zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
40
41The list of checkers tested are hardcoded in the Checkers variable.
42For testing additional checkers, use the SA_ADDITIONAL_CHECKERS environment
43variable. It should contain a comma separated list.
44"""
45import CmpRuns46import SATestUtils as utils47from ProjectMap import DownloadType, ProjectInfo48
49import glob50import logging51import math52import multiprocessing53import os54import plistlib55import shutil56import sys57import threading58import time59import zipfile60
61from queue import Queue62
63# mypy has problems finding InvalidFileException in the module
64# and this is we can shush that false positive
65from plistlib import InvalidFileException # type:ignore66from subprocess import CalledProcessError, check_call67from typing import Dict, IO, List, NamedTuple, Optional, TYPE_CHECKING, Tuple68
69
70###############################################################################
71# Helper functions.
72###############################################################################
73
74
75class StreamToLogger:76def __init__(self, logger: logging.Logger, log_level: int = logging.INFO):77self.logger = logger78self.log_level = log_level79
80def write(self, message: str):81# Rstrip in order not to write an extra newline.82self.logger.log(self.log_level, message.rstrip())83
84def flush(self):85pass86
87def fileno(self) -> int:88return 089
90
91LOCAL = threading.local()92
93
94def init_logger(name: str):95# TODO: use debug levels for VERBOSE messages96logger = logging.getLogger(name)97logger.setLevel(logging.DEBUG)98LOCAL.stdout = StreamToLogger(logger, logging.INFO)99LOCAL.stderr = StreamToLogger(logger, logging.ERROR)100
101
102init_logger("main")103
104
105def stderr(message: str):106LOCAL.stderr.write(message)107
108
109def stdout(message: str):110LOCAL.stdout.write(message)111
112
113logging.basicConfig(format="%(asctime)s:%(levelname)s:%(name)s: %(message)s")114
115
116###############################################################################
117# Configuration setup.
118###############################################################################
119
120
121# Find Clang for static analysis.
122if "CC" in os.environ:123cc_candidate: Optional[str] = os.environ["CC"]124else:125cc_candidate = utils.which("clang", os.environ["PATH"])126if not cc_candidate:127stderr("Error: cannot find 'clang' in PATH")128sys.exit(1)129
130CLANG = cc_candidate131
132# Number of jobs.
133MAX_JOBS = int(math.ceil(multiprocessing.cpu_count() * 0.75))134
135# Names of the project specific scripts.
136# The script that downloads the project.
137DOWNLOAD_SCRIPT = "download_project.sh"138# The script that needs to be executed before the build can start.
139CLEANUP_SCRIPT = "cleanup_run_static_analyzer.sh"140# This is a file containing commands for scan-build.
141BUILD_SCRIPT = "run_static_analyzer.cmd"142
143# A comment in a build script which disables wrapping.
144NO_PREFIX_CMD = "#NOPREFIX"145
146# The log file name.
147LOG_DIR_NAME = "Logs"148BUILD_LOG_NAME = "run_static_analyzer.log"149# Summary file - contains the summary of the failures. Ex: This info can be be
150# displayed when buildbot detects a build failure.
151NUM_OF_FAILURES_IN_SUMMARY = 10152
153# The scan-build result directory.
154OUTPUT_DIR_NAME = "ScanBuildResults"155REF_PREFIX = "Ref"156
157# The name of the directory storing the cached project source. If this
158# directory does not exist, the download script will be executed.
159# That script should create the "CachedSource" directory and download the
160# project source into it.
161CACHED_SOURCE_DIR_NAME = "CachedSource"162
163# The name of the directory containing the source code that will be analyzed.
164# Each time a project is analyzed, a fresh copy of its CachedSource directory
165# will be copied to the PatchedSource directory and then the local patches
166# in PATCHFILE_NAME will be applied (if PATCHFILE_NAME exists).
167PATCHED_SOURCE_DIR_NAME = "PatchedSource"168
169# The name of the patchfile specifying any changes that should be applied
170# to the CachedSource before analyzing.
171PATCHFILE_NAME = "changes_for_analyzer.patch"172
173# The list of checkers used during analyzes.
174# Currently, consists of all the non-experimental checkers, plus a few alpha
175# checkers we don't want to regress on.
176CHECKERS = ",".join(177[178"alpha.unix.SimpleStream",179"alpha.security.taint",180"cplusplus.NewDeleteLeaks",181"core",182"cplusplus",183"deadcode",184"security",185"unix",186"osx",187"nullability",188]189)
190
191VERBOSE = 0192
193
194###############################################################################
195# Test harness logic.
196###############################################################################
197
198
199def run_cleanup_script(directory: str, build_log_file: IO):200"""201Run pre-processing script if any.
202"""
203cwd = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)204script_path = os.path.join(directory, CLEANUP_SCRIPT)205
206utils.run_script(207script_path,208build_log_file,209cwd,210out=LOCAL.stdout,211err=LOCAL.stderr,212verbose=VERBOSE,213)214
215
216class TestInfo(NamedTuple):217"""218Information about a project and settings for its analysis.
219"""
220
221project: ProjectInfo222override_compiler: bool = False223extra_analyzer_config: str = ""224extra_checkers: str = ""225is_reference_build: bool = False226strictness: int = 0227
228
229# typing package doesn't have a separate type for Queue, but has a generic stub
230# We still want to have a type-safe checked project queue, for this reason,
231# we specify generic type for mypy.
232#
233# It is a common workaround for this situation:
234# https://mypy.readthedocs.io/en/stable/common_issues.html#using-classes-that-are-generic-in-stubs-but-not-at-runtime
235if TYPE_CHECKING:236TestQueue = Queue[TestInfo] # this is only processed by mypy237else:238TestQueue = Queue # this will be executed at runtime239
240
241class RegressionTester:242"""243A component aggregating all of the project testing.
244"""
245
246def __init__(247self,248jobs: int,249projects: List[ProjectInfo],250override_compiler: bool,251extra_analyzer_config: str,252extra_checkers: str,253regenerate: bool,254strictness: bool,255):256self.jobs = jobs257self.projects = projects258self.override_compiler = override_compiler259self.extra_analyzer_config = extra_analyzer_config260self.extra_checkers = extra_checkers261self.regenerate = regenerate262self.strictness = strictness263
264def test_all(self) -> bool:265projects_to_test: List[TestInfo] = []266
267# Test the projects.268for project in self.projects:269projects_to_test.append(270TestInfo(271project,272self.override_compiler,273self.extra_analyzer_config,274self.extra_checkers,275self.regenerate,276self.strictness,277)278)279if self.jobs <= 1:280return self._single_threaded_test_all(projects_to_test)281else:282return self._multi_threaded_test_all(projects_to_test)283
284def _single_threaded_test_all(self, projects_to_test: List[TestInfo]) -> bool:285"""286Run all projects.
287:return: whether tests have passed.
288"""
289success = True290for project_info in projects_to_test:291tester = ProjectTester(project_info)292success &= tester.test()293return success294
295def _multi_threaded_test_all(self, projects_to_test: List[TestInfo]) -> bool:296"""297Run each project in a separate thread.
298
299This is OK despite GIL, as testing is blocked
300on launching external processes.
301
302:return: whether tests have passed.
303"""
304tasks_queue = TestQueue()305
306for project_info in projects_to_test:307tasks_queue.put(project_info)308
309results_differ = threading.Event()310failure_flag = threading.Event()311
312for _ in range(self.jobs):313T = TestProjectThread(tasks_queue, results_differ, failure_flag)314T.start()315
316# Required to handle Ctrl-C gracefully.317while tasks_queue.unfinished_tasks:318time.sleep(0.1) # Seconds.319if failure_flag.is_set():320stderr("Test runner crashed\n")321sys.exit(1)322return not results_differ.is_set()323
324
325class ProjectTester:326"""327A component aggregating testing for one project.
328"""
329
330def __init__(self, test_info: TestInfo, silent: bool = False):331self.project = test_info.project332self.override_compiler = test_info.override_compiler333self.extra_analyzer_config = test_info.extra_analyzer_config334self.extra_checkers = test_info.extra_checkers335self.is_reference_build = test_info.is_reference_build336self.strictness = test_info.strictness337self.silent = silent338
339def test(self) -> bool:340"""341Test a given project.
342:return tests_passed: Whether tests have passed according
343to the :param strictness: criteria.
344"""
345if not self.project.enabled:346self.out(f" \n\n--- Skipping disabled project {self.project.name}\n")347return True348
349self.out(f" \n\n--- Building project {self.project.name}\n")350
351start_time = time.time()352
353project_dir = self.get_project_dir()354self.vout(f" Build directory: {project_dir}.\n")355
356# Set the build results directory.357output_dir = self.get_output_dir()358
359self.build(project_dir, output_dir)360check_build(output_dir)361
362if self.is_reference_build:363cleanup_reference_results(output_dir)364passed = True365else:366passed = run_cmp_results(project_dir, self.strictness)367
368self.out(369f"Completed tests for project {self.project.name} "370f"(time: {time.time() - start_time:.2f}).\n"371)372
373return passed374
375def get_project_dir(self) -> str:376return os.path.join(os.path.abspath(os.curdir), self.project.name)377
378def get_output_dir(self) -> str:379if self.is_reference_build:380dirname = REF_PREFIX + OUTPUT_DIR_NAME381else:382dirname = OUTPUT_DIR_NAME383
384return os.path.join(self.get_project_dir(), dirname)385
386def build(self, directory: str, output_dir: str) -> Tuple[float, int]:387build_log_path = get_build_log_path(output_dir)388
389self.out(f"Log file: {build_log_path}\n")390self.out(f"Output directory: {output_dir}\n")391
392remove_log_file(output_dir)393
394# Clean up scan build results.395if os.path.exists(output_dir):396self.vout(f" Removing old results: {output_dir}\n")397
398shutil.rmtree(output_dir)399
400assert not os.path.exists(output_dir)401os.makedirs(os.path.join(output_dir, LOG_DIR_NAME))402
403# Build and analyze the project.404with open(build_log_path, "w+") as build_log_file:405if self.project.mode == 1:406self._download_and_patch(directory, build_log_file)407run_cleanup_script(directory, build_log_file)408build_time, memory = self.scan_build(409directory, output_dir, build_log_file410)411else:412build_time, memory = self.analyze_preprocessed(directory, output_dir)413
414if self.is_reference_build:415run_cleanup_script(directory, build_log_file)416normalize_reference_results(directory, output_dir, self.project.mode)417
418self.out(419f"Build complete (time: {utils.time_to_str(build_time)}, "420f"peak memory: {utils.memory_to_str(memory)}). "421f"See the log for more details: {build_log_path}\n"422)423
424return build_time, memory425
426def scan_build(427self, directory: str, output_dir: str, build_log_file: IO428) -> Tuple[float, int]:429"""430Build the project with scan-build by reading in the commands and
431prefixing them with the scan-build options.
432"""
433build_script_path = os.path.join(directory, BUILD_SCRIPT)434if not os.path.exists(build_script_path):435stderr(f"Error: build script is not defined: " f"{build_script_path}\n")436sys.exit(1)437
438all_checkers = CHECKERS439if "SA_ADDITIONAL_CHECKERS" in os.environ:440all_checkers = all_checkers + "," + os.environ["SA_ADDITIONAL_CHECKERS"]441if self.extra_checkers != "":442all_checkers += "," + self.extra_checkers443
444# Run scan-build from within the patched source directory.445cwd = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)446
447options = f"--use-analyzer '{CLANG}' "448options += f"-plist-html -o '{output_dir}' "449options += f"-enable-checker {all_checkers} "450options += "--keep-empty "451options += f"-analyzer-config '{self.generate_config()}' "452
453if self.override_compiler:454options += "--override-compiler "455
456extra_env: Dict[str, str] = {}457
458execution_time = 0.0459peak_memory = 0460
461try:462command_file = open(build_script_path, "r")463command_prefix = "scan-build " + options + " "464
465for command in command_file:466command = command.strip()467
468if len(command) == 0:469continue470
471# Custom analyzer invocation specified by project.472# Communicate required information using environment variables473# instead.474if command == NO_PREFIX_CMD:475command_prefix = ""476extra_env["OUTPUT"] = output_dir477extra_env["CC"] = CLANG478extra_env["ANALYZER_CONFIG"] = self.generate_config()479continue480
481if command.startswith("#"):482continue483
484# If using 'make', auto imply a -jX argument485# to speed up analysis. xcodebuild will486# automatically use the maximum number of cores.487if (488command.startswith("make ") or command == "make"489) and "-j" not in command:490command += f" -j{MAX_JOBS}"491
492command_to_run = command_prefix + command493
494self.vout(f" Executing: {command_to_run}\n")495
496time, mem = utils.check_and_measure_call(497command_to_run,498cwd=cwd,499stderr=build_log_file,500stdout=build_log_file,501env=dict(os.environ, **extra_env),502shell=True,503)504
505execution_time += time506peak_memory = max(peak_memory, mem)507
508except CalledProcessError:509stderr("Error: scan-build failed. Its output was: \n")510build_log_file.seek(0)511shutil.copyfileobj(build_log_file, LOCAL.stderr)512sys.exit(1)513
514return execution_time, peak_memory515
516def analyze_preprocessed(517self, directory: str, output_dir: str518) -> Tuple[float, int]:519"""520Run analysis on a set of preprocessed files.
521"""
522if os.path.exists(os.path.join(directory, BUILD_SCRIPT)):523stderr(524f"Error: The preprocessed files project "525f"should not contain {BUILD_SCRIPT}\n"526)527raise Exception()528
529prefix = CLANG + " --analyze "530
531prefix += "--analyzer-output plist "532prefix += " -Xclang -analyzer-checker=" + CHECKERS533prefix += " -fcxx-exceptions -fblocks "534prefix += " -Xclang -analyzer-config "535prefix += f"-Xclang {self.generate_config()} "536
537if self.project.mode == 2:538prefix += "-std=c++11 "539
540plist_path = os.path.join(directory, output_dir, "date")541fail_path = os.path.join(plist_path, "failures")542os.makedirs(fail_path)543
544execution_time = 0.0545peak_memory = 0546
547for full_file_name in glob.glob(directory + "/*"):548file_name = os.path.basename(full_file_name)549failed = False550
551# Only run the analyzes on supported files.552if utils.has_no_extension(file_name):553continue554if not utils.is_valid_single_input_file(file_name):555stderr(f"Error: Invalid single input file {full_file_name}.\n")556raise Exception()557
558# Build and call the analyzer command.559plist_basename = os.path.join(plist_path, file_name)560output_option = f"-o '{plist_basename}.plist' "561command = f"{prefix}{output_option}'{file_name}'"562
563log_path = os.path.join(fail_path, file_name + ".stderr.txt")564with open(log_path, "w+") as log_file:565try:566self.vout(f" Executing: {command}\n")567
568time, mem = utils.check_and_measure_call(569command,570cwd=directory,571stderr=log_file,572stdout=log_file,573shell=True,574)575
576execution_time += time577peak_memory = max(peak_memory, mem)578
579except CalledProcessError as e:580stderr(581f"Error: Analyzes of {full_file_name} failed. "582f"See {log_file.name} for details. "583f"Error code {e.returncode}.\n"584)585failed = True586
587# If command did not fail, erase the log file.588if not failed:589os.remove(log_file.name)590
591return execution_time, peak_memory592
593def generate_config(self) -> str:594out = "serialize-stats=true,stable-report-filename=true"595
596if self.extra_analyzer_config:597out += "," + self.extra_analyzer_config598
599return out600
601def _download_and_patch(self, directory: str, build_log_file: IO):602"""603Download the project and apply the local patchfile if it exists.
604"""
605cached_source = os.path.join(directory, CACHED_SOURCE_DIR_NAME)606
607# If the we don't already have the cached source, run the project's608# download script to download it.609if not os.path.exists(cached_source):610self._download(directory, build_log_file)611if not os.path.exists(cached_source):612stderr(f"Error: '{cached_source}' not found after download.\n")613exit(1)614
615patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)616
617# Remove potentially stale patched source.618if os.path.exists(patched_source):619shutil.rmtree(patched_source)620
621# Copy the cached source and apply any patches to the copy.622shutil.copytree(cached_source, patched_source, symlinks=True)623self._apply_patch(directory, build_log_file)624
625def _download(self, directory: str, build_log_file: IO):626"""627Run the script to download the project, if it exists.
628"""
629if self.project.source == DownloadType.GIT:630self._download_from_git(directory, build_log_file)631elif self.project.source == DownloadType.ZIP:632self._unpack_zip(directory, build_log_file)633elif self.project.source == DownloadType.SCRIPT:634self._run_download_script(directory, build_log_file)635else:636raise ValueError(637f"Unknown source type '{self.project.source}' is found "638f"for the '{self.project.name}' project"639)640
641def _download_from_git(self, directory: str, build_log_file: IO):642repo = self.project.origin643cached_source = os.path.join(directory, CACHED_SOURCE_DIR_NAME)644
645check_call(646f"git clone --recursive {repo} {cached_source}",647cwd=directory,648stderr=build_log_file,649stdout=build_log_file,650shell=True,651)652check_call(653f"git checkout --quiet {self.project.commit}",654cwd=cached_source,655stderr=build_log_file,656stdout=build_log_file,657shell=True,658)659
660def _unpack_zip(self, directory: str, build_log_file: IO):661zip_files = list(glob.glob(directory + "/*.zip"))662
663if len(zip_files) == 0:664raise ValueError(665f"Couldn't find any zip files to unpack for the "666f"'{self.project.name}' project"667)668
669if len(zip_files) > 1:670raise ValueError(671f"Couldn't decide which of the zip files ({zip_files}) "672f"for the '{self.project.name}' project to unpack"673)674
675with zipfile.ZipFile(zip_files[0], "r") as zip_file:676zip_file.extractall(os.path.join(directory, CACHED_SOURCE_DIR_NAME))677
678@staticmethod679def _run_download_script(directory: str, build_log_file: IO):680script_path = os.path.join(directory, DOWNLOAD_SCRIPT)681utils.run_script(682script_path,683build_log_file,684directory,685out=LOCAL.stdout,686err=LOCAL.stderr,687verbose=VERBOSE,688)689
690def _apply_patch(self, directory: str, build_log_file: IO):691patchfile_path = os.path.join(directory, PATCHFILE_NAME)692patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)693
694if not os.path.exists(patchfile_path):695self.out(" No local patches.\n")696return697
698self.out(" Applying patch.\n")699try:700check_call(701f"patch -p1 < '{patchfile_path}'",702cwd=patched_source,703stderr=build_log_file,704stdout=build_log_file,705shell=True,706)707
708except CalledProcessError:709stderr(f"Error: Patch failed. " f"See {build_log_file.name} for details.\n")710sys.exit(1)711
712def out(self, what: str):713if not self.silent:714stdout(what)715
716def vout(self, what: str):717if VERBOSE >= 1:718self.out(what)719
720
721class TestProjectThread(threading.Thread):722def __init__(723self,724tasks_queue: TestQueue,725results_differ: threading.Event,726failure_flag: threading.Event,727):728"""729:param results_differ: Used to signify that results differ from
730the canonical ones.
731:param failure_flag: Used to signify a failure during the run.
732"""
733self.tasks_queue = tasks_queue734self.results_differ = results_differ735self.failure_flag = failure_flag736super().__init__()737
738# Needed to gracefully handle interrupts with Ctrl-C739self.daemon = True740
741def run(self):742while not self.tasks_queue.empty():743try:744test_info = self.tasks_queue.get()745init_logger(test_info.project.name)746
747tester = ProjectTester(test_info)748if not tester.test():749self.results_differ.set()750
751self.tasks_queue.task_done()752
753except BaseException:754self.failure_flag.set()755raise756
757
758###############################################################################
759# Utility functions.
760###############################################################################
761
762
763def check_build(output_dir: str):764"""765Given the scan-build output directory, checks if the build failed
766(by searching for the failures directories). If there are failures, it
767creates a summary file in the output directory.
768
769"""
770# Check if there are failures.771failures = glob.glob(output_dir + "/*/failures/*.stderr.txt")772total_failed = len(failures)773
774if total_failed == 0:775clean_up_empty_plists(output_dir)776clean_up_empty_folders(output_dir)777
778plists = glob.glob(output_dir + "/*/*.plist")779stdout(780f"Number of bug reports "781f"(non-empty plist files) produced: {len(plists)}\n"782)783return784
785stderr("Error: analysis failed.\n")786stderr(f"Total of {total_failed} failures discovered.\n")787
788if total_failed > NUM_OF_FAILURES_IN_SUMMARY:789stderr(f"See the first {NUM_OF_FAILURES_IN_SUMMARY} below.\n")790
791for index, failed_log_path in enumerate(failures, start=1):792if index >= NUM_OF_FAILURES_IN_SUMMARY:793break794
795stderr(f"\n-- Error #{index} -----------\n")796
797with open(failed_log_path, "r") as failed_log:798shutil.copyfileobj(failed_log, LOCAL.stdout)799
800if total_failed > NUM_OF_FAILURES_IN_SUMMARY:801stderr("See the results folder for more.")802
803sys.exit(1)804
805
806def cleanup_reference_results(output_dir: str):807"""808Delete html, css, and js files from reference results. These can
809include multiple copies of the benchmark source and so get very large.
810"""
811extensions = ["html", "css", "js"]812
813for extension in extensions:814for file_to_rm in glob.glob(f"{output_dir}/*/*.{extension}"):815file_to_rm = os.path.join(output_dir, file_to_rm)816os.remove(file_to_rm)817
818# Remove the log file. It leaks absolute path names.819remove_log_file(output_dir)820
821
822def run_cmp_results(directory: str, strictness: int = 0) -> bool:823"""824Compare the warnings produced by scan-build.
825strictness defines the success criteria for the test:
8260 - success if there are no crashes or analyzer failure.
8271 - success if there are no difference in the number of reported bugs.
8282 - success if all the bug reports are identical.
829
830:return success: Whether tests pass according to the strictness
831criteria.
832"""
833tests_passed = True834start_time = time.time()835
836ref_dir = os.path.join(directory, REF_PREFIX + OUTPUT_DIR_NAME)837new_dir = os.path.join(directory, OUTPUT_DIR_NAME)838
839# We have to go one level down the directory tree.840ref_list = glob.glob(ref_dir + "/*")841new_list = glob.glob(new_dir + "/*")842
843# Log folders are also located in the results dir, so ignore them.844ref_log_dir = os.path.join(ref_dir, LOG_DIR_NAME)845if ref_log_dir in ref_list:846ref_list.remove(ref_log_dir)847new_list.remove(os.path.join(new_dir, LOG_DIR_NAME))848
849if len(ref_list) != len(new_list):850stderr(f"Mismatch in number of results folders: " f"{ref_list} vs {new_list}")851sys.exit(1)852
853# There might be more then one folder underneath - one per each scan-build854# command (Ex: one for configure and one for make).855if len(ref_list) > 1:856# Assume that the corresponding folders have the same names.857ref_list.sort()858new_list.sort()859
860# Iterate and find the differences.861num_diffs = 0862for ref_dir, new_dir in zip(ref_list, new_list):863assert ref_dir != new_dir864
865if VERBOSE >= 1:866stdout(f" Comparing Results: {ref_dir} {new_dir}\n")867
868patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)869
870ref_results = CmpRuns.ResultsDirectory(ref_dir)871new_results = CmpRuns.ResultsDirectory(new_dir, patched_source)872
873# Scan the results, delete empty plist files.874(875num_diffs,876reports_in_ref,877reports_in_new,878) = CmpRuns.dump_scan_build_results_diff(879ref_results, new_results, delete_empty=False, out=LOCAL.stdout880)881
882if num_diffs > 0:883stdout(f"Warning: {num_diffs} differences in diagnostics.\n")884
885if strictness >= 2 and num_diffs > 0:886stdout("Error: Diffs found in strict mode (2).\n")887tests_passed = False888
889elif strictness >= 1 and reports_in_ref != reports_in_new:890stdout("Error: The number of results are different " " strict mode (1).\n")891tests_passed = False892
893stdout(894f"Diagnostic comparison complete " f"(time: {time.time() - start_time:.2f}).\n"895)896
897return tests_passed898
899
900def normalize_reference_results(directory: str, output_dir: str, build_mode: int):901"""902Make the absolute paths relative in the reference results.
903"""
904for dir_path, _, filenames in os.walk(output_dir):905for filename in filenames:906if not filename.endswith("plist"):907continue908
909plist = os.path.join(dir_path, filename)910with open(plist, "rb") as plist_file:911data = plistlib.load(plist_file)912path_prefix = directory913
914if build_mode == 1:915path_prefix = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)916
917paths = [918source[len(path_prefix) + 1 :]919if source.startswith(path_prefix)920else source921for source in data["files"]922]923data["files"] = paths924
925# Remove transient fields which change from run to run.926for diagnostic in data["diagnostics"]:927if "HTMLDiagnostics_files" in diagnostic:928diagnostic.pop("HTMLDiagnostics_files")929
930if "clang_version" in data:931data.pop("clang_version")932
933with open(plist, "wb") as plist_file:934plistlib.dump(data, plist_file)935
936
937def get_build_log_path(output_dir: str) -> str:938return os.path.join(output_dir, LOG_DIR_NAME, BUILD_LOG_NAME)939
940
941def remove_log_file(output_dir: str):942build_log_path = get_build_log_path(output_dir)943
944# Clean up the log file.945if os.path.exists(build_log_path):946if VERBOSE >= 1:947stdout(f" Removing log file: {build_log_path}\n")948
949os.remove(build_log_path)950
951
952def clean_up_empty_plists(output_dir: str):953"""954A plist file is created for each call to the analyzer(each source file).
955We are only interested on the once that have bug reports,
956so delete the rest.
957"""
958for plist in glob.glob(output_dir + "/*/*.plist"):959plist = os.path.join(output_dir, plist)960
961try:962with open(plist, "rb") as plist_file:963data = plistlib.load(plist_file)964# Delete empty reports.965if not data["files"]:966os.remove(plist)967continue968
969except InvalidFileException as e:970stderr(f"Error parsing plist file {plist}: {str(e)}")971continue972
973
974def clean_up_empty_folders(output_dir: str):975"""976Remove empty folders from results, as git would not store them.
977"""
978subdirs = glob.glob(output_dir + "/*")979for subdir in subdirs:980if not os.listdir(subdir):981os.removedirs(subdir)982
983
984if __name__ == "__main__":985print("SATestBuild.py should not be used on its own.")986print("Please use 'SATest.py build' instead")987sys.exit(1)988