From f5a97d7fbbda5ec8ad4ea73a8035d7fa88bbe1da Mon Sep 17 00:00:00 2001 From: Daniel Berlin Date: Thu, 29 Mar 2012 10:33:19 -0400 Subject: [PATCH] Moving notice file generation to a python script (it was getting fairly complicated in bash), update HTML output to have a table of contents, and fix typos/bugs in notice file CSS Change-Id: I51131f594e3bbe63f03165fa42f5ceeaec518543 --- core/Makefile | 75 +------------ tools/generate-notice-files.py | 188 +++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 72 deletions(-) create mode 100755 tools/generate-notice-files.py diff --git a/core/Makefile b/core/Makefile index e005485eb..bb714abfc 100644 --- a/core/Makefile +++ b/core/Makefile @@ -440,78 +440,9 @@ endif # Then we could traverse that without quite as much bash drama. define combine-notice-files $(1) $(2): PRIVATE_MESSAGE := $(3) -$(1) $(2) $(4)/hash-timestamp: PRIVATE_DIR := $(4) -$(4)/hash-timestamp: $(5) $(BUILD_SYSTEM)/Makefile - @echo Finding NOTICE files: $$@ - $$(hide) rm -rf $$@ $$(PRIVATE_DIR)/hash - $$(hide) mkdir -p $$(PRIVATE_DIR)/hash - $$(hide) for file in $$$$(find $$(PRIVATE_DIR)/src -type f); do \ - hash=$$$$($(MD5SUM) $$$$file | sed -e "s/ .*//"); \ - hashfile=$$(PRIVATE_DIR)/hash/$$$$hash; \ - echo $$$$file >> $$$$hashfile; \ - done - $$(hide) touch $$@ -$(1): $(4)/hash-timestamp - @echo Combining NOTICE files: $$@ - $$(hide) mkdir -p $$(dir $$@) - $$(hide) echo $$(PRIVATE_MESSAGE) > $$@ - $$(hide) find $$(PRIVATE_DIR)/hash -type f | xargs cat | sort | \ - sed -e "s:$$(PRIVATE_DIR)/src\(.*\)\.txt: \1:" >> $$@ - $$(hide) echo >> $$@ - $$(hide) echo >> $$@ - $$(hide) echo >> $$@ - $$(hide) for hashfile in $$$$(find $$(PRIVATE_DIR)/hash -type f); do \ - echo "============================================================"\ - >> $$@; \ - echo "Notices for file(s):" >> $$@; \ - cat $$$$hashfile | sort | \ - sed -e "s:$$(PRIVATE_DIR)/src\(.*\)\.txt: \1:" >> \ - $$@; \ - echo "------------------------------------------------------------"\ - >> $$@; \ - echo >> $$@; \ - orig=$$$$(head -n 1 $$$$hashfile); \ - cat $$$$orig >> $$@; \ - echo >> $$@; \ - echo >> $$@; \ - echo >> $$@; \ - done -$(2): $(4)/hash-timestamp - @echo Combining NOTICE files: $$@ - $$(hide) mkdir -p $$(dir $$@) - $$(hide) echo "" > $$@ - $$(hide) echo "" >> $$@ - $$(hide) echo "" >> $$@ - $$(hide) echo "" \ - >> $$@ - $$(hide) for hashfile in $$$$(find $$(PRIVATE_DIR)/hash -type f); do \ - cat $$$$hashfile | sort | \ - sed -e "s:$$(PRIVATE_DIR)/src\(.*\)\.txt: :" >> \ - $$@; \ - echo "" >> $$@; \ - echo >> $$@; \ - echo >> $$@; \ - echo >> $$@; \ - done - $$(hide) echo "
" >> $$@; \ - echo "
Notices for file(s):
" >> $$@; \ - echo "
" >> $$@; \ - cat $$$$hashfile | sort | \ - sed -e "s:$$(PRIVATE_DIR)/src\(.*\)\.txt: \1
:" >> $$@; \ - echo "
" >> $$@; \ - echo >> $$@; \ - orig=$$$$(head -n 1 $$$$hashfile); \ - echo "
" >> $$@; \
-			cat $$$$orig | sed -e "s/\&/\&/g" | sed -e "s//\>/g" >> $$@; \
-			echo "
" >> $$@; \ - echo "
" >> $$@ - $$(hide) echo "" >> $$@ +$(1) $(2): PRIVATE_DIR := $(4) +$(1) $(2): $(5) $(BUILD_SYSTEM)/Makefile build/tools/generate-notice-files.py + build/tools/generate-notice-files.py $(1) $(2) $$(PRIVATE_MESSAGE) $$(PRIVATE_DIR)/src notice_files: $(1) $(2) endef diff --git a/tools/generate-notice-files.py b/tools/generate-notice-files.py new file mode 100755 index 000000000..4571b70df --- /dev/null +++ b/tools/generate-notice-files.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python +# +# Copyright (C) 2012 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Usage: generate-notice-files [plain text output file] [html output file] [file title] [directory of notices] + +Generate the Android notice files, including both text and html files. + +-h to display this usage message and exit. +""" +from collections import defaultdict +import getopt +import hashlib +import itertools +import os +import os.path +import re +import sys + +MD5_BLOCKSIZE = 1024 * 1024 +HTML_ESCAPE_TABLE = { + "&": "&", + '"': """, + "'": "'", + ">": ">", + "<": "<", + } + +try: + opts, args = getopt.getopt(sys.argv[1:], "h") +except getopt.GetoptError, err: + print str(err) + print __doc__ + sys.exit(2) + +for o, a in opts: + if o == "-h": + print __doc__ + sys.exit(2) + else: + print >> sys.stderr, "unhandled option %s" % (o,) + +if len(args) != 4: + print """need exactly four arguments, the two output files, the file title + and the directory containing notices, not %d""" % (len(args),) + print __doc__ + sys.exit(1) + +def hexify(s): + return ("%02x"*len(s)) % tuple(map(ord, s)) + +def md5sum(filename): + """Calculate an MD5 of the file given by FILENAME, + and return hex digest as a string. + Output should be compatible with md5sum command""" + + f = open(filename, "rb") + sum = hashlib.md5() + while 1: + block = f.read(MD5_BLOCKSIZE) + if not block: + break + sum.update(block) + f.close() + return hexify(sum.digest()) + + +def html_escape(text): + """Produce entities within text.""" + return "".join(HTML_ESCAPE_TABLE.get(c,c) for c in text) + +HTML_OUTPUT_CSS=""" + +""" + +def combine_notice_files_html(file_hash, input_dir, output_filename): + """Combine notice files in FILE_HASH and output a HTML version to OUTPUT_FILENAME.""" + + SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") + + # Set up a filename to row id table (anchors inside tables don't work in + # most browsers, but href's to table row ids do) + id_table = {} + id_count = 0 + for value in file_hash.values(): + for filename in value: + id_table[filename] = id_count + id_count += 1 + + # Open the output file, and output the header pieces + output_file = open(output_filename, "wb") + + print >> output_file, "" + print >> output_file, HTML_OUTPUT_CSS + print >> output_file, '' + + # Output our table of contents + print >> output_file, '
' + print >> output_file, "" + print >> output_file, "
" + # Output the individual notice file lists + print >>output_file, '' + for value in file_hash.values(): + print >> output_file, '" + print >> output_file + print >> output_file + print >> output_file + + # Finish off the file output + print >> output_file, "
' % id_table.get(value[0]) + print >> output_file, '
Notices for file(s):
' + print >> output_file, '
' + for filename in sorted(value): + print >> output_file, "%s
" % (SRC_DIR_STRIP_RE.sub(r"\1", filename)) + print >> output_file, "
" + print >> output_file + print >> output_file, '
'
+        print >> output_file, html_escape(open(value[0]).read())
+        print >> output_file, "
" + print >> output_file, "
" + print >> output_file, "" + output_file.close() + +def combine_notice_files_text(file_hash, input_dir, output_filename, file_title): + """Combine notice files in FILE_HASH and output a text version to OUTPUT_FILENAME.""" + + SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") + output_file = open(output_filename, "wb") + print >> output_file, file_title + for value in file_hash.values(): + print >> output_file, "============================================================" + print >> output_file, "Notices for file(s):" + for filename in sorted(value): + print >> output_file, SRC_DIR_STRIP_RE.sub(r"\1", filename) + print >> output_file, "------------------------------------------------------------" + print >> output_file, open(value[0]).read() + output_file.close() + +def main(args): + txt_output_file = args[0] + html_output_file = args[1] + file_title = args[2] + + # Find all the notice files and md5 them + input_dir = os.path.normpath(args[3]) + files_with_same_hash = defaultdict(list) + for root, dir, files in os.walk(input_dir): + for file in files: + if file.endswith(".txt"): + filename = os.path.join(root, file) + file_md5sum = md5sum(filename) + files_with_same_hash[file_md5sum].append(filename) + + + print "Combining NOTICE files into HTML" + combine_notice_files_html(files_with_same_hash, input_dir, html_output_file) + print "Combining NOTICE files into text" + combine_notice_files_text(files_with_same_hash, input_dir, txt_output_file, file_title) + +if __name__ == "__main__": + main(args)