p96705831
/
platform_build
forked from openkylin/platform_build
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge \\"Count warnings per project and dump out CSV format.\\" am: 053c54b554 

am: d59fe53668

Change-Id: I8eef3954ca560feabf6fcba46a89a3aaa18d88c3
This commit is contained in:
Chih-hung Hsieh 2016-07-22 01:05:15 +00:00 committed by android-build-merger
parent 0b6d82b798 d59fe53668
commit 94861af0ec
1 changed files with 168 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

221
tools/warn.py
View File

@ -6,6 +6,10 @@ import sys
import re import re
parser = argparse.ArgumentParser(description='Convert a build log into HTML') parser = argparse.ArgumentParser(description='Convert a build log into HTML')
parser.add_argument('--gencsv',
help='Generate a CSV file with number of various warnings',
action="store_true",
default=False)
parser.add_argument('--url', parser.add_argument('--url',
help='Root URL of an Android source code tree prefixed ' help='Root URL of an Android source code tree prefixed '
'before files in warnings') 'before files in warnings')
@ -1789,6 +1793,54 @@ warnpatterns = [
'patterns':[r".*: warning: .+"] }, 'patterns':[r".*: warning: .+"] },
] ]
# A list of [project_name, file_path_pattern].
# project_name should not contain comma, to be used in CSV output.
projectlist = [
['art', r"(^|.*/)art/.*: warning:"],
['bionic', r"(^|.*/)bionic/.*: warning:"],
['bootable', r"(^|.*/)bootable/.*: warning:"],
['build', r"(^|.*/)build/.*: warning:"],
['cts', r"(^|.*/)cts/.*: warning:"],
['dalvik', r"(^|.*/)dalvik/.*: warning:"],
['developers', r"(^|.*/)developers/.*: warning:"],
['development', r"(^|.*/)development/.*: warning:"],
['device', r"(^|.*/)device/.*: warning:"],
['doc', r"(^|.*/)doc/.*: warning:"],
['external', r"(^|.*/)external/.*: warning:"],
['frameworks', r"(^|.*/)frameworks/.*: warning:"],
['hardware', r"(^|.*/)hardware/.*: warning:"],
['kernel', r"(^|.*/)kernel/.*: warning:"],
['libcore', r"(^|.*/)libcore/.*: warning:"],
['libnativehelper', r"(^|.*/)libnativehelper/.*: warning:"],
['ndk', r"(^|.*/)ndk/.*: warning:"],
['packages', r"(^|.*/)packages/.*: warning:"],
['pdk', r"(^|.*/)pdk/.*: warning:"],
['prebuilts', r"(^|.*/)prebuilts/.*: warning:"],
['system', r"(^|.*/)system/.*: warning:"],
['toolchain', r"(^|.*/)toolchain/.*: warning:"],
['test', r"(^|.*/)test/.*: warning:"],
['tools', r"(^|.*/)tools/.*: warning:"],
['vendor', r"(^|.*/)vendor/.*: warning:"],
['out/obj', r".*/(gen|obj[^/]*)/(include|EXECUTABLES|SHARED_LIBRARIES|STATIC_LIBRARIES)/.*: warning:"],
['other', r".*: warning:"],
]
projectpatterns = []
for p in projectlist:
projectpatterns.append({'description':p[0], 'members':[], 'pattern':re.compile(p[1])})
# Each warning pattern has a dictionary that maps
# a project name to number of warnings in that project.
for w in warnpatterns:
w['projects'] = {}
platformversion = 'unknown'
targetproduct = 'unknown'
targetvariant = 'unknown'
##### Data and functions to dump html file. ##################################
anchor = 0 anchor = 0
cur_row_class = 0 cur_row_class = 0
@ -1837,7 +1889,6 @@ def dumphtmlprologue(title):
output('<title>' + title + '</title>\n') output('<title>' + title + '</title>\n')
output(html_script_style) output(html_script_style)
output('</head>\n<body>\n') output('</head>\n<body>\n')
output('<a name="PageTop">')
output(htmlbig(title)) output(htmlbig(title))
output('<p>\n') output('<p>\n')
@ -1851,12 +1902,16 @@ def tablerow(text):
output(text) output(text)
output('</td></tr>') output('</td></tr>')
def sortwarnings():
for i in warnpatterns:
i['members'] = sorted(set(i['members']))
# dump some stats about total number of warnings and such # dump some stats about total number of warnings and such
def dumpstats(): def dumpstats():
known = 0 known = 0
unknown = 0 unknown = 0
sortwarnings()
for i in warnpatterns: for i in warnpatterns:
i['members'] = sorted(set(i['members']))
if i['severity'] == severity.UNKNOWN: if i['severity'] == severity.UNKNOWN:
unknown += len(i['members']) unknown += len(i['members'])
elif i['severity'] != severity.SKIP: elif i['severity'] != severity.SKIP:
@ -1961,17 +2016,28 @@ def dumpcategory(cat):
output('</table></div>\n') output('</table></div>\n')
def findproject(line):
for p in projectpatterns:
if p['pattern'].match(line):
return p['description']
return '???'
def classifywarning(line): def classifywarning(line):
for i in warnpatterns: for i in warnpatterns:
for cpat in i['compiledpatterns']: for cpat in i['compiledpatterns']:
if cpat.match(line): if cpat.match(line):
i['members'].append(line) i['members'].append(line)
pname = findproject(line)
if pname in i['projects']:
i['projects'][pname] += 1
else:
i['projects'][pname] = 1
return return
else: else:
# If we end up here, there was a problem parsing the log # If we end up here, there was a problem parsing the log
# probably caused by 'make -j' mixing the output from # probably caused by 'make -j' mixing the output from
# 2 or more concurrent compiles # 2 or more concurrent compiles
pass pass
# precompiling every pattern speeds up parsing by about 30x # precompiling every pattern speeds up parsing by about 30x
def compilepatterns(): def compilepatterns():
@ -1980,54 +2046,103 @@ def compilepatterns():
for pat in i['patterns']: for pat in i['patterns']:
i['compiledpatterns'].append(re.compile(pat)) i['compiledpatterns'].append(re.compile(pat))
infile = open(args.buildlog, 'r') def parseinputfile():
warnings = [] global platformversion
global targetproduct
global targetvariant
infile = open(args.buildlog, 'r')
linecounter = 0
platformversion = 'unknown' warningpattern = re.compile('.* warning:.*')
targetproduct = 'unknown' compilepatterns()
targetvariant = 'unknown'
linecounter = 0
warningpattern = re.compile('.* warning:.*') # read the log file and classify all the warnings
compilepatterns() warninglines = set()
for line in infile:
# read the log file and classify all the warnings # replace fancy quotes with plain ol' quotes
warninglines = set() line = line.replace("‘", "'");
for line in infile: line = line.replace("’", "'");
# replace fancy quotes with plain ol' quotes if warningpattern.match(line):
line = line.replace("‘", "'"); if line not in warninglines:
line = line.replace("’", "'"); classifywarning(line)
if warningpattern.match(line): warninglines.add(line)
if line not in warninglines: else:
classifywarning(line) # save a little bit of time by only doing this for the first few lines
warninglines.add(line) if linecounter < 50:
else: linecounter +=1
# save a little bit of time by only doing this for the first few lines m = re.search('(?<=^PLATFORM_VERSION=).*', line)
if linecounter < 50: if m != None:
linecounter +=1 platformversion = m.group(0)
m = re.search('(?<=^PLATFORM_VERSION=).*', line) m = re.search('(?<=^TARGET_PRODUCT=).*', line)
if m != None: if m != None:
platformversion = m.group(0) targetproduct = m.group(0)
m = re.search('(?<=^TARGET_PRODUCT=).*', line) m = re.search('(?<=^TARGET_BUILD_VARIANT=).*', line)
if m != None: if m != None:
targetproduct = m.group(0) targetvariant = m.group(0)
m = re.search('(?<=^TARGET_BUILD_VARIANT=).*', line)
if m != None:
targetvariant = m.group(0)
# dump the html output to stdout # dump the html output to stdout
dumphtmlprologue('Warnings for ' + platformversion + ' - ' + targetproduct + ' - ' + targetvariant) def dumphtml():
dumpstats() dumphtmlprologue('Warnings for ' + platformversion + ' - ' + targetproduct + ' - ' + targetvariant)
# sort table based on number of members once dumpstats has deduplicated the dumpstats()
# members. # sort table based on number of members once dumpstats has deduplicated the
warnpatterns.sort(reverse=True, key=lambda i: len(i['members'])) # members.
dumpseverity(severity.FIXMENOW) warnpatterns.sort(reverse=True, key=lambda i: len(i['members']))
dumpseverity(severity.HIGH) dumpseverity(severity.FIXMENOW)
dumpseverity(severity.MEDIUM) dumpseverity(severity.HIGH)
dumpseverity(severity.LOW) dumpseverity(severity.MEDIUM)
dumpseverity(severity.TIDY) dumpseverity(severity.LOW)
dumpseverity(severity.HARMLESS) dumpseverity(severity.TIDY)
dumpseverity(severity.UNKNOWN) dumpseverity(severity.HARMLESS)
dumpfixed() dumpseverity(severity.UNKNOWN)
dumphtmlepilogue() dumpfixed()
dumphtmlepilogue()
##### Functions to count warnings and dump csv file. #########################
def descriptionforcsv(cat):
if cat['description'] == '':
return '?'
return cat['description']
def stringforcsv(s):
if ',' in s:
return '"{}"'.format(s)
return s
def countseverity(sev, kind):
sum = 0
for i in warnpatterns:
if i['severity'] == sev and len(i['members']) > 0:
n = len(i['members'])
sum += n
warning = stringforcsv(kind + ': ' + descriptionforcsv(i))
print '{},,{}'.format(n, warning)
# print number of warnings for each project, ordered by project name.
projects = i['projects'].keys()
projects.sort()
for p in projects:
print '{},{},{}'.format(i['projects'][p], p, warning)
print '{},,{}'.format(sum, kind + ' warnings')
return sum
# dump number of warnings in csv format to stdout
def dumpcsv():
sortwarnings()
total = 0
total += countseverity(severity.FIXMENOW, 'FixNow')
total += countseverity(severity.HIGH, 'High')
total += countseverity(severity.MEDIUM, 'Medium')
total += countseverity(severity.LOW, 'Low')
total += countseverity(severity.TIDY, 'Tidy')
total += countseverity(severity.HARMLESS, 'Harmless')
total += countseverity(severity.UNKNOWN, 'Unknown')
print '{},,{}'.format(total, 'All warnings')
parseinputfile()
if args.gencsv:
dumpcsv()
else:
dumphtml()