[analyzer] Static Analyzer Qualification Infrastructure: Scripts to support basic testing of the analyzer on external projects. This can be used as a basis for setting up a buildbot.
llvm-svn: 141337
This commit is contained in:
parent
9776e438cf
commit
f0c4116202
|
@ -0,0 +1,71 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Static Analyzer qualification infrastructure: adding a new project to
|
||||
the Repository Directory.
|
||||
|
||||
Add a new project for testing: build it and add to the Project Map file.
|
||||
Assumes it's being run from the Repository Directory.
|
||||
The project directory should be added inside the Repository Directory and
|
||||
have the same name as the project ID
|
||||
|
||||
The project should use the following files for set up:
|
||||
- pre_run_static_analyzer.sh - prepare the build environment.
|
||||
Ex: make clean can be a part of it.
|
||||
- run_static_analyzer.cmd - a list of commands to run through scan-build.
|
||||
Each command should be on a separate line.
|
||||
Choose from: configure, make, xcodebuild
|
||||
"""
|
||||
import SATestBuild
|
||||
|
||||
import os
|
||||
import csv
|
||||
import sys
|
||||
|
||||
# Add a new project for testing: build it and add to the Project Map file.
|
||||
# Params:
|
||||
# Dir is the directory where the sources are.
|
||||
# ID is a short string used to identify a project.
|
||||
def addNewProject(ID) :
|
||||
CurDir = os.path.abspath(os.curdir)
|
||||
Dir = SATestBuild.getProjectDir(ID)
|
||||
if not os.path.exists(Dir):
|
||||
print "Error: Project directory is missing: %s" % Dir
|
||||
sys.exit(-1)
|
||||
|
||||
# Build the project.
|
||||
SATestBuild.testProject(ID, True, Dir)
|
||||
|
||||
# Add the project ID to the project map.
|
||||
ProjectMapPath = os.path.join(CurDir, SATestBuild.ProjectMapFile)
|
||||
if os.path.exists(ProjectMapPath):
|
||||
PMapFile = open(ProjectMapPath, "r+b")
|
||||
else:
|
||||
print "Warning: Creating the Project Map file!!"
|
||||
PMapFile = open(ProjectMapPath, "w+b")
|
||||
try:
|
||||
PMapReader = csv.reader(PMapFile)
|
||||
for I in PMapReader:
|
||||
IID = I[0]
|
||||
if ID == IID:
|
||||
print >> sys.stderr, 'Warning: Project with ID \'', ID, \
|
||||
'\' already exists.'
|
||||
sys.exit(-1)
|
||||
|
||||
PMapWriter = csv.writer(PMapFile)
|
||||
PMapWriter.writerow( (ID, Dir) );
|
||||
finally:
|
||||
PMapFile.close()
|
||||
|
||||
print "The project map is updated: ", ProjectMapPath
|
||||
|
||||
|
||||
# TODO: Add an option not to build.
|
||||
# TODO: Set the path to the Repository directory.
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 2:
|
||||
print >> sys.stderr, 'Usage: ', sys.argv[0],\
|
||||
'[project ID]'
|
||||
sys.exit(-1)
|
||||
|
||||
addNewProject(sys.argv[1])
|
|
@ -0,0 +1,307 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Static Analyzer qualification infrastructure.
|
||||
|
||||
The goal is to test the analyzer against different projects, check for failures,
|
||||
compare results, and measure performance.
|
||||
|
||||
Repository Directory will contain sources of the projects as well as the
|
||||
information on how to build them and the expected output.
|
||||
Repository Directory structure:
|
||||
- ProjectMap file
|
||||
- Historical Performance Data
|
||||
- Project Dir1
|
||||
- ReferenceOutput
|
||||
- Project Dir2
|
||||
- ReferenceOutput
|
||||
..
|
||||
|
||||
To test the build of the analyzer one would:
|
||||
- Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that
|
||||
the build directory does not pollute the repository to min network traffic).
|
||||
- Build all projects, until error. Produce logs to report errors.
|
||||
- Compare results.
|
||||
|
||||
The files which should be kept around for failure investigations:
|
||||
RepositoryCopy/Project DirI/ScanBuildResults
|
||||
RepositoryCopy/Project DirI/run_static_analyzer.log
|
||||
|
||||
Assumptions (TODO: shouldn't need to assume these.):
|
||||
The script is being run from the Repository Directory.
|
||||
The compiler for scan-build is in the PATH.
|
||||
export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH
|
||||
|
||||
For more logging, set the env variables:
|
||||
zaks:TI zaks$ export CCC_ANALYZER_LOG=1
|
||||
zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
|
||||
"""
|
||||
import CmpRuns
|
||||
|
||||
import os
|
||||
import csv
|
||||
import sys
|
||||
import glob
|
||||
import shutil
|
||||
import time
|
||||
import plistlib
|
||||
from subprocess import check_call
|
||||
|
||||
# Project map stores info about all the "registered" projects.
|
||||
ProjectMapFile = "projectMap.csv"
|
||||
|
||||
# Names of the project specific scripts.
|
||||
# The script that needs to be executed before the build can start.
|
||||
PreprocessScript = "pre_run_static_analyzer.sh"
|
||||
# This is a file containing commands for scan-build.
|
||||
BuildScript = "run_static_analyzer.cmd"
|
||||
|
||||
# The log file name.
|
||||
BuildLogName = "run_static_analyzer.log"
|
||||
# Summary file - contains the summary of the failures. Ex: This info can be be
|
||||
# displayed when buildbot detects a build failure.
|
||||
NumOfFailuresInSummary = 10
|
||||
FailuresSummaryFileName = "failures.txt"
|
||||
# Summary of the result diffs.
|
||||
DiffsSummaryFileName = "diffs.txt"
|
||||
|
||||
# The scan-build result directory.
|
||||
SBOutputDirName = "ScanBuildResults"
|
||||
SBOutputDirReferencePrefix = "Ref"
|
||||
|
||||
Verbose = 1
|
||||
|
||||
def getProjectMapPath():
|
||||
ProjectMapPath = os.path.join(os.path.abspath(os.curdir),
|
||||
ProjectMapFile)
|
||||
if not os.path.exists(ProjectMapPath):
|
||||
print "Error: Cannot find the Project Map file " + ProjectMapPath +\
|
||||
"\nRunning script for the wrong directory?"
|
||||
sys.exit(-1)
|
||||
return ProjectMapPath
|
||||
|
||||
def getProjectDir(ID):
|
||||
return os.path.join(os.path.abspath(os.curdir), ID)
|
||||
|
||||
# Run pre-processing script if any.
|
||||
def runPreProcessingScript(Dir, PBuildLogFile):
|
||||
ScriptPath = os.path.join(Dir, PreprocessScript)
|
||||
if os.path.exists(ScriptPath):
|
||||
try:
|
||||
if Verbose == 1:
|
||||
print " Executing: %s" % (ScriptPath,)
|
||||
check_call("chmod +x %s" % ScriptPath, cwd = Dir,
|
||||
stderr=PBuildLogFile,
|
||||
stdout=PBuildLogFile,
|
||||
shell=True)
|
||||
check_call(ScriptPath, cwd = Dir, stderr=PBuildLogFile,
|
||||
stdout=PBuildLogFile,
|
||||
shell=True)
|
||||
except:
|
||||
print "Error: The pre-processing step failed. See ", \
|
||||
PBuildLogFile.name, " for details."
|
||||
sys.exit(-1)
|
||||
|
||||
# Build the project with scan-build by reading in the commands and
|
||||
# prefixing them with the scan-build options.
|
||||
def runScanBuild(Dir, SBOutputDir, PBuildLogFile):
|
||||
BuildScriptPath = os.path.join(Dir, BuildScript)
|
||||
if not os.path.exists(BuildScriptPath):
|
||||
print "Error: build script is not defined: %s" % BuildScriptPath
|
||||
sys.exit(-1)
|
||||
SBOptions = "-plist -o " + SBOutputDir + " "
|
||||
SBOptions += "-enable-checker core,deadcode.DeadStores"
|
||||
try:
|
||||
SBCommandFile = open(BuildScriptPath, "r")
|
||||
SBPrefix = "scan-build " + SBOptions + " "
|
||||
for Command in SBCommandFile:
|
||||
SBCommand = SBPrefix + Command
|
||||
if Verbose == 1:
|
||||
print " Executing: %s" % (SBCommand,)
|
||||
check_call(SBCommand, cwd = Dir, stderr=PBuildLogFile,
|
||||
stdout=PBuildLogFile,
|
||||
shell=True)
|
||||
except:
|
||||
print "Error: scan-build failed. See ",PBuildLogFile.name,\
|
||||
" for details."
|
||||
sys.exit(-1)
|
||||
|
||||
def buildProject(Dir, SBOutputDir):
|
||||
TBegin = time.time()
|
||||
|
||||
BuildLogPath = os.path.join(Dir, BuildLogName)
|
||||
print "Log file: %s" % (BuildLogPath,)
|
||||
|
||||
# Clean up the log file.
|
||||
if (os.path.exists(BuildLogPath)) :
|
||||
RmCommand = "rm " + BuildLogPath
|
||||
if Verbose == 1:
|
||||
print " Executing: %s." % (RmCommand,)
|
||||
check_call(RmCommand, shell=True)
|
||||
|
||||
# Open the log file.
|
||||
PBuildLogFile = open(BuildLogPath, "wb+")
|
||||
try:
|
||||
# Clean up scan build results.
|
||||
if (os.path.exists(SBOutputDir)) :
|
||||
RmCommand = "rm -r " + SBOutputDir
|
||||
if Verbose == 1:
|
||||
print " Executing: %s" % (RmCommand,)
|
||||
check_call(RmCommand, stderr=PBuildLogFile,
|
||||
stdout=PBuildLogFile, shell=True)
|
||||
|
||||
runPreProcessingScript(Dir, PBuildLogFile)
|
||||
runScanBuild(Dir, SBOutputDir, PBuildLogFile)
|
||||
finally:
|
||||
PBuildLogFile.close()
|
||||
|
||||
print "Build complete (time: %.2f). See the log for more details: %s" % \
|
||||
((time.time()-TBegin), BuildLogPath)
|
||||
|
||||
# A plist file is created for each call to the analyzer(each source file).
|
||||
# We are only interested on the once that have bug reports, so delete the rest.
|
||||
def CleanUpEmptyPlists(SBOutputDir):
|
||||
for F in glob.glob(SBOutputDir + "/*/*.plist"):
|
||||
P = os.path.join(SBOutputDir, F)
|
||||
|
||||
Data = plistlib.readPlist(P)
|
||||
# Delete empty reports.
|
||||
if not Data['files']:
|
||||
os.remove(P)
|
||||
continue
|
||||
|
||||
# Given the scan-build output directory, checks if the build failed
|
||||
# (by searching for the failures directories). If there are failures, it
|
||||
# creates a summary file in the output directory.
|
||||
def checkBuild(SBOutputDir):
|
||||
# Check if there are failures.
|
||||
Failures = glob.glob(SBOutputDir + "/*/failures/*.stderr.txt")
|
||||
TotalFailed = len(Failures);
|
||||
if TotalFailed == 0:
|
||||
CleanUpEmptyPlists(SBOutputDir)
|
||||
Plists = glob.glob(SBOutputDir + "/*/*.plist")
|
||||
print "Number of bug reports (non empty plist files) produced: %d" %\
|
||||
len(Plists)
|
||||
return;
|
||||
|
||||
# Create summary file to display when the build fails.
|
||||
SummaryPath = os.path.join(SBOutputDir, FailuresSummaryFileName);
|
||||
if (Verbose > 0):
|
||||
print " Creating the failures summary file %s." % (SummaryPath,)
|
||||
|
||||
SummaryLog = open(SummaryPath, "w+")
|
||||
try:
|
||||
SummaryLog.write("Total of %d failures discovered.\n" % (TotalFailed,))
|
||||
if TotalFailed > NumOfFailuresInSummary:
|
||||
SummaryLog.write("See the first %d below.\n"
|
||||
% (NumOfFailuresInSummary,))
|
||||
# TODO: Add a line "See the results folder for more."
|
||||
|
||||
FailuresCopied = NumOfFailuresInSummary
|
||||
Idx = 0
|
||||
for FailLogPathI in glob.glob(SBOutputDir + "/*/failures/*.stderr.txt"):
|
||||
if Idx >= NumOfFailuresInSummary:
|
||||
break;
|
||||
Idx += 1
|
||||
SummaryLog.write("\n-- Error #%d -----------\n" % (Idx,));
|
||||
FailLogI = open(FailLogPathI, "r");
|
||||
try:
|
||||
shutil.copyfileobj(FailLogI, SummaryLog);
|
||||
finally:
|
||||
FailLogI.close()
|
||||
finally:
|
||||
SummaryLog.close()
|
||||
|
||||
print "Error: Scan-build failed. See ", \
|
||||
os.path.join(SBOutputDir, FailuresSummaryFileName)
|
||||
sys.exit(-1)
|
||||
|
||||
# Auxiliary object to discard stdout.
|
||||
class Discarder(object):
|
||||
def write(self, text):
|
||||
pass # do nothing
|
||||
|
||||
# Compare the warnings produced by scan-build.
|
||||
def runCmpResults(Dir):
|
||||
TBegin = time.time()
|
||||
|
||||
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
|
||||
NewDir = os.path.join(Dir, SBOutputDirName)
|
||||
|
||||
# We have to go one level down the directory tree.
|
||||
RefList = glob.glob(RefDir + "/*")
|
||||
NewList = glob.glob(NewDir + "/*")
|
||||
if len(RefList) == 0 or len(NewList) == 0:
|
||||
return False
|
||||
assert(len(RefList) == len(NewList))
|
||||
|
||||
# There might be more then one folder underneath - one per each scan-build
|
||||
# command (Ex: one for configure and one for make).
|
||||
if (len(RefList) > 1):
|
||||
# Assume that the corresponding folders have the same names.
|
||||
RefList.sort()
|
||||
NewList.sort()
|
||||
|
||||
# Iterate and find the differences.
|
||||
HaveDiffs = False
|
||||
PairList = zip(RefList, NewList)
|
||||
for P in PairList:
|
||||
RefDir = P[0]
|
||||
NewDir = P[1]
|
||||
|
||||
assert(RefDir != NewDir)
|
||||
if Verbose == 1:
|
||||
print " Comparing Results: %s %s" % (RefDir, NewDir)
|
||||
|
||||
DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
|
||||
Opts = CmpRuns.CmpOptions(DiffsPath)
|
||||
# Discard everything coming out of stdout (CmpRun produces a lot of them).
|
||||
OLD_STDOUT = sys.stdout
|
||||
sys.stdout = Discarder()
|
||||
# Scan the results, delete empty plist files.
|
||||
HaveDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False)
|
||||
sys.stdout = OLD_STDOUT
|
||||
if HaveDiffs:
|
||||
print "Warning: difference in diagnostics. See %s" % (DiffsPath,)
|
||||
HaveDiffs=True
|
||||
|
||||
print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin)
|
||||
return HaveDiffs
|
||||
|
||||
def testProject(ID, IsReferenceBuild, Dir=None):
|
||||
TBegin = time.time()
|
||||
|
||||
if Dir is None :
|
||||
Dir = getProjectDir(ID)
|
||||
if Verbose == 1:
|
||||
print " Build directory: %s." % (Dir,)
|
||||
|
||||
# Set the build results directory.
|
||||
if IsReferenceBuild == True :
|
||||
SBOutputDir = os.path.join(Dir, SBOutputDirReferencePrefix + \
|
||||
SBOutputDirName)
|
||||
else :
|
||||
SBOutputDir = os.path.join(Dir, SBOutputDirName)
|
||||
|
||||
buildProject(Dir, SBOutputDir)
|
||||
|
||||
checkBuild(SBOutputDir)
|
||||
|
||||
if IsReferenceBuild == False:
|
||||
runCmpResults(Dir)
|
||||
|
||||
print "Completed tests for project %s (time: %.2f)." % \
|
||||
(ID, (time.time()-TBegin))
|
||||
|
||||
def testAll(IsReferenceBuild=False):
|
||||
PMapFile = open(getProjectMapPath(), "rb")
|
||||
try:
|
||||
PMapReader = csv.reader(PMapFile)
|
||||
for I in PMapReader:
|
||||
print " --- Building project %s" % (I[0],)
|
||||
testProject(I[0], IsReferenceBuild)
|
||||
finally:
|
||||
PMapFile.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
testAll()
|
Loading…
Reference in New Issue