From 3911e6a1c4ec2d968f0ac1fb6dc9f7a19827dfe3 Mon Sep 17 00:00:00 2001 From: Bob Badour Date: Mon, 10 Feb 2020 17:08:47 -0800 Subject: [PATCH] Make a soong-only copy of generate-notice-files.py This is an unchanged copy of build/make/tools/generate-notice-files.py After this change, the make implementation of generate-notice-files.py will change to take a required parameter for $(PRODUCT_OUT) and will only include notices for files built under $(PRODUCT_OUT). Because soong has the ability to walk dependency trees, the soong version will likely change in the future to use that ability causing both versions to move in different directions. After those changes are complete, we can look into factoring out any remaining shared logic. Test: run manually and system image notices checked for changes Change-Id: Id139a66503457615548b46e7996349ca0817e831 --- android/notices.go | 2 +- scripts/generate-notice-files.py | 267 +++++++++++++++++++++++++++++++ scripts/mergenotice.py | 2 +- 3 files changed, 269 insertions(+), 2 deletions(-) create mode 100755 scripts/generate-notice-files.py diff --git a/android/notices.go b/android/notices.go index bf273b544..07cf3e4d0 100644 --- a/android/notices.go +++ b/android/notices.go @@ -22,7 +22,7 @@ import ( func init() { pctx.SourcePathVariable("merge_notices", "build/soong/scripts/mergenotice.py") - pctx.SourcePathVariable("generate_notice", "build/make/tools/generate-notice-files.py") + pctx.SourcePathVariable("generate_notice", "build/soong/scripts/generate-notice-files.py") pctx.HostBinToolVariable("minigzip", "minigzip") } diff --git a/scripts/generate-notice-files.py b/scripts/generate-notice-files.py new file mode 100755 index 000000000..49011b20f --- /dev/null +++ b/scripts/generate-notice-files.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python +# +# Copyright (C) 2012 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Usage: generate-notice-files --text-output [plain text output file] \ + --html-output [html output file] \ + --xml-output [xml output file] \ + -t [file title] -s [directory of notices] + +Generate the Android notice files, including both text and html files. + +-h to display this usage message and exit. +""" +from collections import defaultdict +import argparse +import hashlib +import itertools +import os +import os.path +import re +import sys + +MD5_BLOCKSIZE = 1024 * 1024 +HTML_ESCAPE_TABLE = { + "&": "&", + '"': """, + "'": "'", + ">": ">", + "<": "<", + } + +def hexify(s): + return ("%02x"*len(s)) % tuple(map(ord, s)) + +def md5sum(filename): + """Calculate an MD5 of the file given by FILENAME, + and return hex digest as a string. + Output should be compatible with md5sum command""" + + f = open(filename, "rb") + sum = hashlib.md5() + while 1: + block = f.read(MD5_BLOCKSIZE) + if not block: + break + sum.update(block) + f.close() + return hexify(sum.digest()) + + +def html_escape(text): + """Produce entities within text.""" + return "".join(HTML_ESCAPE_TABLE.get(c,c) for c in text) + +HTML_OUTPUT_CSS=""" + +""" + +def combine_notice_files_html(file_hash, input_dir, output_filename): + """Combine notice files in FILE_HASH and output a HTML version to OUTPUT_FILENAME.""" + + SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") + + # Set up a filename to row id table (anchors inside tables don't work in + # most browsers, but href's to table row ids do) + id_table = {} + id_count = 0 + for value in file_hash: + for filename in value: + id_table[filename] = id_count + id_count += 1 + + # Open the output file, and output the header pieces + output_file = open(output_filename, "wb") + + print >> output_file, "" + print >> output_file, HTML_OUTPUT_CSS + print >> output_file, '' + + # Output our table of contents + print >> output_file, '
' + print >> output_file, "" + print >> output_file, "
" + # Output the individual notice file lists + print >>output_file, '' + for value in file_hash: + print >> output_file, '" + print >> output_file + print >> output_file + print >> output_file + + # Finish off the file output + print >> output_file, "
' % id_table.get(value[0]) + print >> output_file, '
Notices for file(s):
' + print >> output_file, '
' + for filename in value: + print >> output_file, "%s
" % (SRC_DIR_STRIP_RE.sub(r"\1", filename)) + print >> output_file, "
" + print >> output_file + print >> output_file, '
'
+        print >> output_file, html_escape(open(value[0]).read())
+        print >> output_file, "
" + print >> output_file, "
" + print >> output_file, "" + output_file.close() + +def combine_notice_files_text(file_hash, input_dir, output_filename, file_title): + """Combine notice files in FILE_HASH and output a text version to OUTPUT_FILENAME.""" + + SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") + output_file = open(output_filename, "wb") + print >> output_file, file_title + for value in file_hash: + print >> output_file, "============================================================" + print >> output_file, "Notices for file(s):" + for filename in value: + print >> output_file, SRC_DIR_STRIP_RE.sub(r"\1", filename) + print >> output_file, "------------------------------------------------------------" + print >> output_file, open(value[0]).read() + output_file.close() + +def combine_notice_files_xml(files_with_same_hash, input_dir, output_filename): + """Combine notice files in FILE_HASH and output a XML version to OUTPUT_FILENAME.""" + + SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") + + # Set up a filename to row id table (anchors inside tables don't work in + # most browsers, but href's to table row ids do) + id_table = {} + for file_key in files_with_same_hash.keys(): + for filename in files_with_same_hash[file_key]: + id_table[filename] = file_key + + # Open the output file, and output the header pieces + output_file = open(output_filename, "wb") + + print >> output_file, '' + print >> output_file, "" + + # Flatten the list of lists into a single list of filenames + sorted_filenames = sorted(id_table.keys()) + + # Print out a nice table of contents + for filename in sorted_filenames: + stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename) + print >> output_file, '%s' % (id_table.get(filename), stripped_filename) + + print >> output_file + print >> output_file + + processed_file_keys = [] + # Output the individual notice file lists + for filename in sorted_filenames: + file_key = id_table.get(filename) + if file_key in processed_file_keys: + continue + processed_file_keys.append(file_key) + + print >> output_file, '' % (file_key, html_escape(open(filename).read())) + print >> output_file + + # Finish off the file output + print >> output_file, "" + output_file.close() + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--text-output', required=True, + help='The text output file path.') + parser.add_argument( + '--html-output', + help='The html output file path.') + parser.add_argument( + '--xml-output', + help='The xml output file path.') + parser.add_argument( + '-t', '--title', required=True, + help='The file title.') + parser.add_argument( + '-s', '--source-dir', required=True, + help='The directory containing notices.') + parser.add_argument( + '-i', '--included-subdirs', action='append', + help='The sub directories which should be included.') + parser.add_argument( + '-e', '--excluded-subdirs', action='append', + help='The sub directories which should be excluded.') + return parser.parse_args() + +def main(argv): + args = get_args() + + txt_output_file = args.text_output + html_output_file = args.html_output + xml_output_file = args.xml_output + file_title = args.title + included_subdirs = [] + excluded_subdirs = [] + if args.included_subdirs is not None: + included_subdirs = args.included_subdirs + if args.excluded_subdirs is not None: + excluded_subdirs = args.excluded_subdirs + + # Find all the notice files and md5 them + input_dir = os.path.normpath(args.source_dir) + files_with_same_hash = defaultdict(list) + for root, dir, files in os.walk(input_dir): + for file in files: + matched = True + if len(included_subdirs) > 0: + matched = False + for subdir in included_subdirs: + if (root == (input_dir + '/' + subdir) or + root.startswith(input_dir + '/' + subdir + '/')): + matched = True + break + elif len(excluded_subdirs) > 0: + for subdir in excluded_subdirs: + if (root == (input_dir + '/' + subdir) or + root.startswith(input_dir + '/' + subdir + '/')): + matched = False + break + if matched and file.endswith(".txt"): + filename = os.path.join(root, file) + file_md5sum = md5sum(filename) + files_with_same_hash[file_md5sum].append(filename) + + filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(files_with_same_hash.keys())] + + combine_notice_files_text(filesets, input_dir, txt_output_file, file_title) + + if html_output_file is not None: + combine_notice_files_html(filesets, input_dir, html_output_file) + + if xml_output_file is not None: + combine_notice_files_xml(files_with_same_hash, input_dir, xml_output_file) + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/mergenotice.py b/scripts/mergenotice.py index 407ae8cc4..fe990735b 100755 --- a/scripts/mergenotice.py +++ b/scripts/mergenotice.py @@ -16,7 +16,7 @@ # """ Merges input notice files to the output file while ignoring duplicated files -This script shouldn't be confused with build/make/tools/generate-notice-files.py +This script shouldn't be confused with build/soong/scripts/generate-notice-files.py which is responsible for creating the final notice file for all artifacts installed. This script has rather limited scope; it is meant to create a merged notice file for a set of modules that are packaged together, e.g. in an APEX.