diff --git a/tools/warn/chrome_project_list.py b/tools/warn/chrome_project_list.py new file mode 100644 index 000000000..609652267 --- /dev/null +++ b/tools/warn/chrome_project_list.py @@ -0,0 +1,686 @@ +# python3 +"""Clang_Tidy_Warn Project List data for Chrome. + +This file stores the Chrome project_list used in warn.py and +its dependencies. It has been put into this file for easier navigation and +unification of the Chrome and Android warn.py. +""" + + +def create_pattern(pattern): + return [pattern, '(^|.*/)' + pattern + '/.*: warning:'] + + +# A list of [project_name, file_path_pattern]. +project_list = [ + create_pattern('android_webview'), + create_pattern('apps'), + create_pattern('ash/app_list'), + create_pattern('ash/public'), + create_pattern('ash/assistant'), + create_pattern('ash/display'), + create_pattern('ash/resources'), + create_pattern('ash/login'), + create_pattern('ash/system'), + create_pattern('ash/wm'), + create_pattern('ash/shelf'), + create_pattern('ash'), + create_pattern('base/trace_event'), + create_pattern('base/debug'), + create_pattern('base/third_party'), + create_pattern('base/files'), + create_pattern('base/test'), + create_pattern('base/util'), + create_pattern('base/task'), + create_pattern('base/metrics'), + create_pattern('base/strings'), + create_pattern('base/memory'), + create_pattern('base'), + create_pattern('build'), + create_pattern('build_overrides'), + create_pattern('buildtools'), + create_pattern('cc'), + create_pattern('chrome/services'), + create_pattern('chrome/app'), + create_pattern('chrome/renderer'), + create_pattern('chrome/test'), + create_pattern('chrome/common/safe_browsing'), + create_pattern('chrome/common/importer'), + create_pattern('chrome/common/media_router'), + create_pattern('chrome/common/extensions'), + create_pattern('chrome/common'), + create_pattern('chrome/browser/sync_file_system'), + create_pattern('chrome/browser/safe_browsing'), + create_pattern('chrome/browser/download'), + create_pattern('chrome/browser/ui'), + create_pattern('chrome/browser/supervised_user'), + create_pattern('chrome/browser/search'), + create_pattern('chrome/browser/browsing_data'), + create_pattern('chrome/browser/predictors'), + create_pattern('chrome/browser/net'), + create_pattern('chrome/browser/devtools'), + create_pattern('chrome/browser/resource_coordinator'), + create_pattern('chrome/browser/page_load_metrics'), + create_pattern('chrome/browser/extensions'), + create_pattern('chrome/browser/ssl'), + create_pattern('chrome/browser/printing'), + create_pattern('chrome/browser/profiles'), + create_pattern('chrome/browser/chromeos'), + create_pattern('chrome/browser/performance_manager'), + create_pattern('chrome/browser/metrics'), + create_pattern('chrome/browser/component_updater'), + create_pattern('chrome/browser/media'), + create_pattern('chrome/browser/notifications'), + create_pattern('chrome/browser/web_applications'), + create_pattern('chrome/browser/media_galleries'), + create_pattern('chrome/browser'), + create_pattern('chrome'), + create_pattern('chromecast'), + create_pattern('chromeos/services'), + create_pattern('chromeos/dbus'), + create_pattern('chromeos/assistant'), + create_pattern('chromeos/components'), + create_pattern('chromeos/settings'), + create_pattern('chromeos/constants'), + create_pattern('chromeos/network'), + create_pattern('chromeos'), + create_pattern('cloud_print'), + create_pattern('components/crash'), + create_pattern('components/subresource_filter'), + create_pattern('components/invalidation'), + create_pattern('components/autofill'), + create_pattern('components/onc'), + create_pattern('components/arc'), + create_pattern('components/safe_browsing'), + create_pattern('components/services'), + create_pattern('components/cast_channel'), + create_pattern('components/download'), + create_pattern('components/feed'), + create_pattern('components/offline_pages'), + create_pattern('components/bookmarks'), + create_pattern('components/cloud_devices'), + create_pattern('components/mirroring'), + create_pattern('components/spellcheck'), + create_pattern('components/viz'), + create_pattern('components/gcm_driver'), + create_pattern('components/ntp_snippets'), + create_pattern('components/translate'), + create_pattern('components/search_engines'), + create_pattern('components/background_task_scheduler'), + create_pattern('components/signin'), + create_pattern('components/chromeos_camera'), + create_pattern('components/reading_list'), + create_pattern('components/assist_ranker'), + create_pattern('components/payments'), + create_pattern('components/feedback'), + create_pattern('components/ui_devtools'), + create_pattern('components/password_manager'), + create_pattern('components/omnibox'), + create_pattern('components/content_settings'), + create_pattern('components/dom_distiller'), + create_pattern('components/nacl'), + create_pattern('components/metrics'), + create_pattern('components/policy'), + create_pattern('components/optimization_guide'), + create_pattern('components/exo'), + create_pattern('components/update_client'), + create_pattern('components/data_reduction_proxy'), + create_pattern('components/sync'), + create_pattern('components/drive'), + create_pattern('components/variations'), + create_pattern('components/history'), + create_pattern('components/webcrypto'), + create_pattern('components'), + create_pattern('content/public'), + create_pattern('content/renderer'), + create_pattern('content/test'), + create_pattern('content/common'), + create_pattern('content/browser'), + create_pattern('content/zygote'), + create_pattern('content'), + create_pattern('courgette'), + create_pattern('crypto'), + create_pattern('dbus'), + create_pattern('device/base'), + create_pattern('device/vr'), + create_pattern('device/gamepad'), + create_pattern('device/test'), + create_pattern('device/fido'), + create_pattern('device/bluetooth'), + create_pattern('device'), + create_pattern('docs'), + create_pattern('extensions/docs'), + create_pattern('extensions/components'), + create_pattern('extensions/buildflags'), + create_pattern('extensions/renderer'), + create_pattern('extensions/test'), + create_pattern('extensions/common'), + create_pattern('extensions/shell'), + create_pattern('extensions/browser'), + create_pattern('extensions/strings'), + create_pattern('extensions'), + create_pattern('fuchsia'), + create_pattern('gin'), + create_pattern('google_apis'), + create_pattern('google_update'), + create_pattern('gpu/perftests'), + create_pattern('gpu/GLES2'), + create_pattern('gpu/command_buffer'), + create_pattern('gpu/tools'), + create_pattern('gpu/gles2_conform_support'), + create_pattern('gpu/ipc'), + create_pattern('gpu/khronos_glcts_support'), + create_pattern('gpu'), + create_pattern('headless'), + create_pattern('infra'), + create_pattern('ipc'), + create_pattern('jingle'), + create_pattern('media'), + create_pattern('mojo'), + create_pattern('native_client'), + create_pattern('ative_client_sdk'), + create_pattern('net'), + create_pattern('out'), + create_pattern('pdf'), + create_pattern('ppapi'), + create_pattern('printing'), + create_pattern('remoting'), + create_pattern('rlz'), + create_pattern('sandbox'), + create_pattern('services/audio'), + create_pattern('services/content'), + create_pattern('services/data_decoder'), + create_pattern('services/device'), + create_pattern('services/file'), + create_pattern('services/identity'), + create_pattern('services/image_annotation'), + create_pattern('services/media_session'), + create_pattern('services/metrics'), + create_pattern('services/network'), + create_pattern('services/preferences'), + create_pattern('services/proxy_resolver'), + create_pattern('services/resource_coordinator'), + create_pattern('services/service_manager'), + create_pattern('services/shape_detection'), + create_pattern('services/strings'), + create_pattern('services/test'), + create_pattern('services/tracing'), + create_pattern('services/video_capture'), + create_pattern('services/viz'), + create_pattern('services/ws'), + create_pattern('services'), + create_pattern('skia/config'), + create_pattern('skia/ext'), + create_pattern('skia/public'), + create_pattern('skia/tools'), + create_pattern('skia'), + create_pattern('sql'), + create_pattern('storage'), + create_pattern('styleguide'), + create_pattern('testing'), + create_pattern('third_party/Python-Markdown'), + create_pattern('third_party/SPIRV-Tools'), + create_pattern('third_party/abseil-cpp'), + create_pattern('third_party/accessibility-audit'), + create_pattern('third_party/accessibility_test_framework'), + create_pattern('third_party/adobe'), + create_pattern('third_party/afl'), + create_pattern('third_party/android_build_tools'), + create_pattern('third_party/android_crazy_linker'), + create_pattern('third_party/android_data_chart'), + create_pattern('third_party/android_deps'), + create_pattern('third_party/android_media'), + create_pattern('third_party/android_ndk'), + create_pattern('third_party/android_opengl'), + create_pattern('third_party/android_platform'), + create_pattern('third_party/android_protobuf'), + create_pattern('third_party/android_sdk'), + create_pattern('third_party/android_support_test_runner'), + create_pattern('third_party/android_swipe_refresh'), + create_pattern('third_party/android_system_sdk'), + create_pattern('third_party/android_tools'), + create_pattern('third_party/angle'), + create_pattern('third_party/apache-mac'), + create_pattern('third_party/apache-portable-runtime'), + create_pattern('third_party/apache-win32'), + create_pattern('third_party/apk-patch-size-estimator'), + create_pattern('third_party/apple_apsl'), + create_pattern('third_party/arcore-android-sdk'), + create_pattern('third_party/ashmem'), + create_pattern('third_party/auto'), + create_pattern('third_party/axe-core'), + create_pattern('third_party/bazel'), + create_pattern('third_party/binutils'), + create_pattern('third_party/bison'), + create_pattern('third_party/blanketjs'), + create_pattern('third_party/blink/common'), + create_pattern('third_party/blink/manual_tests'), + create_pattern('third_party/blink/perf_tests'), + create_pattern('third_party/blink/public/common'), + create_pattern('third_party/blink/public/default_100_percent'), + create_pattern('third_party/blink/public/default_200_percent'), + create_pattern('third_party/blink/public/platform'), + create_pattern('third_party/blink/public/mojom/ad_tagging'), + create_pattern('third_party/blink/public/mojom/app_banner'), + create_pattern('third_party/blink/public/mojom/appcache'), + create_pattern('third_party/blink/public/mojom/array_buffer'), + create_pattern('third_party/blink/public/mojom/associated_interfaces'), + create_pattern('third_party/blink/public/mojom/autoplay'), + create_pattern('third_party/blink/public/mojom/background_fetch'), + create_pattern('third_party/blink/public/mojom/background_sync'), + create_pattern('third_party/blink/public/mojom/badging'), + create_pattern('third_party/blink/public/mojom/blob'), + create_pattern('third_party/blink/public/mojom/bluetooth'), + create_pattern('third_party/blink/public/mojom/broadcastchannel'), + create_pattern('third_party/blink/public/mojom/cache_storage'), + create_pattern('third_party/blink/public/mojom/choosers'), + create_pattern('third_party/blink/public/mojom/clipboard'), + create_pattern('third_party/blink/public/mojom/commit_result'), + create_pattern('third_party/blink/public/mojom/contacts'), + create_pattern('third_party/blink/public/mojom/cookie_store'), + create_pattern('third_party/blink/public/mojom/crash'), + create_pattern('third_party/blink/public/mojom/credentialmanager'), + create_pattern('third_party/blink/public/mojom/csp'), + create_pattern('third_party/blink/public/mojom/devtools'), + create_pattern('third_party/blink/public/mojom/document_metadata'), + create_pattern('third_party/blink/public/mojom/dom_storage'), + create_pattern('third_party/blink/public/mojom/dwrite_font_proxy'), + create_pattern('third_party/blink/public/mojom/feature_policy'), + create_pattern('third_party/blink/public/mojom/fetch'), + create_pattern('third_party/blink/public/mojom/file'), + create_pattern('third_party/blink/public/mojom/filesystem'), + create_pattern('third_party/blink/public/mojom/font_unique_name_lookup'), + create_pattern('third_party/blink/public/mojom/frame'), + create_pattern('third_party/blink/public/mojom/frame_sinks'), + create_pattern('third_party/blink/public/mojom/geolocation'), + create_pattern('third_party/blink/public/mojom/hyphenation'), + create_pattern('third_party/blink/public/mojom/idle'), + create_pattern('third_party/blink/public/mojom/indexeddb'), + create_pattern('third_party/blink/public/mojom/input'), + create_pattern('third_party/blink/public/mojom/insecure_input'), + create_pattern('third_party/blink/public/mojom/installation'), + create_pattern('third_party/blink/public/mojom/installedapp'), + create_pattern('third_party/blink/public/mojom/keyboard_lock'), + create_pattern('third_party/blink/public/mojom/leak_detector'), + create_pattern('third_party/blink/public/mojom/loader'), + create_pattern('third_party/blink/public/mojom/locks'), + create_pattern('third_party/blink/public/mojom/manifest'), + create_pattern('third_party/blink/public/mojom/media_controls'), + create_pattern('third_party/blink/public/mojom/mediasession'), + create_pattern('third_party/blink/public/mojom/mediastream'), + create_pattern('third_party/blink/public/mojom/messaging'), + create_pattern('third_party/blink/public/mojom/mime'), + create_pattern('third_party/blink/public/mojom/native_file_system'), + create_pattern('third_party/blink/public/mojom/net'), + create_pattern('third_party/blink/public/mojom/notifications'), + create_pattern('third_party/blink/public/mojom/oom_intervention'), + create_pattern('third_party/blink/public/mojom/page'), + create_pattern('third_party/blink/public/mojom/payments'), + create_pattern('third_party/blink/public/mojom/permissions'), + create_pattern('third_party/blink/public/mojom/picture_in_picture'), + create_pattern('third_party/blink/public/mojom/plugins'), + create_pattern('third_party/blink/public/mojom/portal'), + create_pattern('third_party/blink/public/mojom/presentation'), + create_pattern('third_party/blink/public/mojom/push_messaging'), + create_pattern('third_party/blink/public/mojom/quota'), + create_pattern('third_party/blink/public/mojom/remote_objects'), + create_pattern('third_party/blink/public/mojom/reporting'), + create_pattern('third_party/blink/public/mojom/script'), + create_pattern('third_party/blink/public/mojom/selection_menu'), + create_pattern('third_party/blink/public/mojom/serial'), + create_pattern('third_party/blink/public/mojom/service_worker'), + create_pattern('third_party/blink/public/mojom/site_engagement'), + create_pattern('third_party/blink/public/mojom/sms'), + create_pattern('third_party/blink/public/mojom/speech'), + create_pattern('third_party/blink/public/mojom/ukm'), + create_pattern('third_party/blink/public/mojom/unhandled_tap_notifier'), + create_pattern('third_party/blink/public/mojom/usb'), + create_pattern('third_party/blink/public/mojom/use_counter'), + create_pattern('third_party/blink/public/mojom/user_agent'), + create_pattern('third_party/blink/public/mojom/wake_lock'), + create_pattern('third_party/blink/public/mojom/web_client_hints'), + create_pattern('third_party/blink/public/mojom/web_feature'), + create_pattern('third_party/blink/public/mojom/webaudio'), + create_pattern('third_party/blink/public/mojom/webauthn'), + create_pattern('third_party/blink/public/mojom/webdatabase'), + create_pattern('third_party/blink/public/mojom/webshare'), + create_pattern('third_party/blink/public/mojom/window_features'), + create_pattern('third_party/blink/public/mojom/worker'), + create_pattern('third_party/blink/public/web'), + create_pattern('third_party/blink/renderer/bindings'), + create_pattern('third_party/blink/renderer/build'), + create_pattern('third_party/blink/renderer/controller'), + create_pattern('third_party/blink/renderer/core/accessibility'), + create_pattern('third_party/blink/renderer/core/animation'), + create_pattern('third_party/blink/renderer/core/aom'), + create_pattern('third_party/blink/renderer/core/clipboard'), + create_pattern('third_party/blink/renderer/core/content_capture'), + create_pattern('third_party/blink/renderer/core/context_features'), + create_pattern('third_party/blink/renderer/core/css'), + create_pattern('third_party/blink/renderer/core/display_lock'), + create_pattern('third_party/blink/renderer/core/dom'), + create_pattern('third_party/blink/renderer/core/editing'), + create_pattern('third_party/blink/renderer/core/events'), + create_pattern('third_party/blink/renderer/core/execution_context'), + create_pattern('third_party/blink/renderer/core/exported'), + create_pattern('third_party/blink/renderer/core/feature_policy'), + create_pattern('third_party/blink/renderer/core/fetch'), + create_pattern('third_party/blink/renderer/core/fileapi'), + create_pattern('third_party/blink/renderer/core/frame'), + create_pattern('third_party/blink/renderer/core/fullscreen'), + create_pattern('third_party/blink/renderer/core/geometry'), + create_pattern('third_party/blink/renderer/core/html'), + create_pattern('third_party/blink/renderer/core/imagebitmap'), + create_pattern('third_party/blink/renderer/core/input'), + create_pattern('third_party/blink/renderer/core/inspector'), + create_pattern('third_party/blink/renderer/core/intersection_observer'), + create_pattern('third_party/blink/renderer/core/invisible_dom'), + create_pattern('third_party/blink/renderer/core/layout'), + create_pattern('third_party/blink/renderer/core/loader'), + create_pattern('third_party/blink/renderer/core/messaging'), + create_pattern('third_party/blink/renderer/core/mojo'), + create_pattern('third_party/blink/renderer/core/offscreencanvas'), + create_pattern('third_party/blink/renderer/core/origin_trials'), + create_pattern('third_party/blink/renderer/core/page'), + create_pattern('third_party/blink/renderer/core/paint'), + create_pattern('third_party/blink/renderer/core/probe'), + create_pattern('third_party/blink/renderer/core/resize_observer'), + create_pattern('third_party/blink/renderer/core/scheduler'), + create_pattern('third_party/blink/renderer/core/script'), + create_pattern('third_party/blink/renderer/core/scroll'), + create_pattern('third_party/blink/renderer/core/streams'), + create_pattern('third_party/blink/renderer/core/style'), + create_pattern('third_party/blink/renderer/core/svg'), + create_pattern('third_party/blink/renderer/core/testing'), + create_pattern('third_party/blink/renderer/core/timezone'), + create_pattern('third_party/blink/renderer/core/timing'), + create_pattern('third_party/blink/renderer/core/trustedtypes'), + create_pattern('third_party/blink/renderer/core/typed_arrays'), + create_pattern('third_party/blink/renderer/core/url'), + create_pattern('third_party/blink/renderer/core/win'), + create_pattern('third_party/blink/renderer/core/workers'), + create_pattern('third_party/blink/renderer/core/xml'), + create_pattern('third_party/blink/renderer/core/xmlhttprequest'), + create_pattern('third_party/blink/renderer/devtools'), + create_pattern('third_party/blink/renderer/modules'), + create_pattern('third_party/blink/renderer/platform'), + create_pattern('third_party/blink/tools'), + create_pattern('third_party/blink/web_tests'), + create_pattern('third_party/boringssl'), + create_pattern('third_party/bouncycastle'), + create_pattern('third_party/breakpad'), + create_pattern('third_party/brotli'), + create_pattern('third_party/bspatch'), + create_pattern('third_party/byte_buddy'), + create_pattern('third_party/cacheinvalidation'), + create_pattern('third_party/catapult'), + create_pattern('third_party/cct_dynamic_module'), + create_pattern('third_party/ced'), + create_pattern('third_party/chaijs'), + create_pattern('third_party/checkstyle'), + create_pattern('third_party/chromevox'), + create_pattern('third_party/chromite'), + create_pattern('third_party/cld_3'), + create_pattern('third_party/closure_compiler'), + create_pattern('third_party/colorama'), + create_pattern('third_party/crashpad'), + create_pattern('third_party/crc32c'), + create_pattern('third_party/cros_system_api'), + create_pattern('third_party/custom_tabs_client'), + create_pattern('third_party/d3'), + create_pattern('third_party/dav1d'), + create_pattern('third_party/dawn'), + create_pattern('third_party/decklink'), + create_pattern('third_party/depot_tools'), + create_pattern('third_party/devscripts'), + create_pattern('third_party/devtools-node-modules'), + create_pattern('third_party/dom_distiller_js'), + create_pattern('third_party/elfutils'), + create_pattern('third_party/emoji-segmenter'), + create_pattern('third_party/errorprone'), + create_pattern('third_party/espresso'), + create_pattern('third_party/expat'), + create_pattern('third_party/feed'), + create_pattern('third_party/ffmpeg'), + create_pattern('third_party/flac'), + create_pattern('third_party/flatbuffers'), + create_pattern('third_party/flot'), + create_pattern('third_party/fontconfig'), + create_pattern('third_party/freetype'), + create_pattern('third_party/fuchsia-sdk'), + create_pattern('third_party/gestures'), + create_pattern('third_party/gif_player'), + create_pattern('third_party/glfw'), + create_pattern('third_party/glslang'), + create_pattern('third_party/gnu_binutils'), + create_pattern('third_party/google-truth'), + create_pattern('third_party/google_android_play_core'), + create_pattern('third_party/google_appengine_cloudstorage'), + create_pattern('third_party/google_input_tools'), + create_pattern('third_party/google_toolbox_for_mac'), + create_pattern('third_party/google_trust_services'), + create_pattern('third_party/googletest'), + create_pattern('third_party/gperf'), + create_pattern('third_party/gradle_wrapper'), + create_pattern('third_party/grpc'), + create_pattern('third_party/gson'), + create_pattern('third_party/guava'), + create_pattern('third_party/gvr-android-keyboard'), + create_pattern('third_party/gvr-android-sdk'), + create_pattern('third_party/hamcrest'), + create_pattern('third_party/harfbuzz-ng'), + create_pattern('third_party/hunspell'), + create_pattern('third_party/hunspell_dictionaries'), + create_pattern('third_party/iaccessible2'), + create_pattern('third_party/iccjpeg'), + create_pattern('third_party/icu/android'), + create_pattern('third_party/icu/android_small'), + create_pattern('third_party/icu/cast'), + create_pattern('third_party/icu/chromeos'), + create_pattern('third_party/icu/common'), + create_pattern('third_party/icu/filters'), + create_pattern('third_party/icu/flutter'), + create_pattern('third_party/icu/fuzzers'), + create_pattern('third_party/icu/ios'), + create_pattern('third_party/icu/patches'), + create_pattern('third_party/icu/scripts'), + create_pattern('third_party/icu/source'), + create_pattern('third_party/icu/tzres'), + create_pattern('third_party/icu4j'), + create_pattern('third_party/ijar'), + create_pattern('third_party/ink'), + create_pattern('third_party/inspector_protocol'), + create_pattern('third_party/instrumented_libraries'), + create_pattern('third_party/intellij'), + create_pattern('third_party/isimpledom'), + create_pattern('third_party/jacoco'), + create_pattern('third_party/jinja2'), + create_pattern('third_party/jsoncpp'), + create_pattern('third_party/jsr-305'), + create_pattern('third_party/jstemplate'), + create_pattern('third_party/junit'), + create_pattern('third_party/khronos'), + create_pattern('third_party/lcov'), + create_pattern('third_party/leveldatabase'), + create_pattern('third_party/libFuzzer'), + create_pattern('third_party/libXNVCtrl'), + create_pattern('third_party/libaddressinput'), + create_pattern('third_party/libaom'), + create_pattern('third_party/libcxx-pretty-printers'), + create_pattern('third_party/libdrm'), + create_pattern('third_party/libevdev'), + create_pattern('third_party/libjingle_xmpp'), + create_pattern('third_party/libjpeg'), + create_pattern('third_party/libjpeg_turbo'), + create_pattern('third_party/liblouis'), + create_pattern('third_party/libovr'), + create_pattern('third_party/libphonenumber'), + create_pattern('third_party/libpng'), + create_pattern('third_party/libprotobuf-mutator'), + create_pattern('third_party/libsecret'), + create_pattern('third_party/libsrtp'), + create_pattern('third_party/libsync'), + create_pattern('third_party/libudev'), + create_pattern('third_party/libusb'), + create_pattern('third_party/libvpx'), + create_pattern('third_party/libwebm'), + create_pattern('third_party/libwebp'), + create_pattern('third_party/libxml'), + create_pattern('third_party/libxslt'), + create_pattern('third_party/libyuv'), + create_pattern('third_party/lighttpd'), + create_pattern('third_party/logilab'), + create_pattern('third_party/lss'), + create_pattern('third_party/lzma_sdk'), + create_pattern('third_party/mach_override'), + create_pattern('third_party/markdown'), + create_pattern('third_party/markupsafe'), + create_pattern('third_party/material_design_icons'), + create_pattern('third_party/mesa_headers'), + create_pattern('third_party/metrics_proto'), + create_pattern('third_party/microsoft_webauthn'), + create_pattern('third_party/mingw-w64'), + create_pattern('third_party/minigbm'), + create_pattern('third_party/minizip'), + create_pattern('third_party/mocha'), + create_pattern('third_party/mockito'), + create_pattern('third_party/modp_b64'), + create_pattern('third_party/motemplate'), + create_pattern('third_party/mozilla'), + create_pattern('third_party/nacl_sdk_binaries'), + create_pattern('third_party/nasm'), + create_pattern('third_party/netty-tcnative'), + create_pattern('third_party/netty4'), + create_pattern('third_party/node'), + create_pattern('third_party/nvml'), + create_pattern('third_party/objenesis'), + create_pattern('third_party/ocmock'), + create_pattern('third_party/openh264'), + create_pattern('third_party/openscreen'), + create_pattern('third_party/openvr'), + create_pattern('third_party/opus'), + create_pattern('third_party/ots'), + create_pattern('third_party/ow2_asm'), + create_pattern('third_party/pdfium'), + create_pattern('third_party/pefile'), + create_pattern('third_party/perfetto'), + create_pattern('third_party/perl'), + create_pattern('third_party/pexpect'), + create_pattern('third_party/pffft'), + create_pattern('third_party/ply'), + create_pattern('third_party/polymer'), + create_pattern('third_party/proguard'), + create_pattern('third_party/protobuf'), + create_pattern('third_party/protoc_javalite'), + create_pattern('third_party/pycoverage'), + create_pattern('third_party/pyelftools'), + create_pattern('third_party/pyjson5'), + create_pattern('third_party/pylint'), + create_pattern('third_party/pymock'), + create_pattern('third_party/pystache'), + create_pattern('third_party/pywebsocket'), + create_pattern('third_party/qcms'), + create_pattern('third_party/quic_trace'), + create_pattern('third_party/qunit'), + create_pattern('third_party/r8'), + create_pattern('third_party/re2'), + create_pattern('third_party/requests'), + create_pattern('third_party/rnnoise'), + create_pattern('third_party/robolectric'), + create_pattern('third_party/s2cellid'), + create_pattern('third_party/sfntly'), + create_pattern('third_party/shaderc'), + create_pattern('third_party/simplejson'), + create_pattern('third_party/sinonjs'), + create_pattern('third_party/skia'), + create_pattern('third_party/smhasher'), + create_pattern('third_party/snappy'), + create_pattern('third_party/speech-dispatcher'), + create_pattern('third_party/spirv-cross'), + create_pattern('third_party/spirv-headers'), + create_pattern('third_party/sqlite'), + create_pattern('third_party/sqlite4java'), + create_pattern('third_party/sudden_motion_sensor'), + create_pattern('third_party/swiftshader'), + create_pattern('third_party/tcmalloc'), + create_pattern('third_party/test_fonts'), + create_pattern('third_party/tlslite'), + create_pattern('third_party/ub-uiautomator'), + create_pattern('third_party/unrar'), + create_pattern('third_party/usb_ids'), + create_pattern('third_party/usrsctp'), + create_pattern('third_party/v4l-utils'), + create_pattern('third_party/vulkan'), + create_pattern('third_party/wayland'), + create_pattern('third_party/wayland-protocols'), + create_pattern('third_party/wds'), + create_pattern('third_party/web-animations-js'), + create_pattern('third_party/webdriver'), + create_pattern('third_party/webgl'), + create_pattern('third_party/webrtc'), + create_pattern('third_party/webrtc_overrides'), + create_pattern('third_party/webxr_test_pages'), + create_pattern('third_party/widevine'), + create_pattern('third_party/win_build_output'), + create_pattern('third_party/woff2'), + create_pattern('third_party/wtl'), + create_pattern('third_party/xdg-utils'), + create_pattern('third_party/xstream'), + create_pattern('third_party/yasm'), + create_pattern('third_party/zlib'), + create_pattern('tools'), + create_pattern('ui/accelerated_widget_mac'), + create_pattern('ui/accessibility'), + create_pattern('ui/android'), + create_pattern('ui/aura'), + create_pattern('ui/aura_extra'), + create_pattern('ui/base'), + create_pattern('ui/chromeos'), + create_pattern('ui/compositor'), + create_pattern('ui/compositor_extra'), + create_pattern('ui/content_accelerators'), + create_pattern('ui/display'), + create_pattern('ui/events'), + create_pattern('ui/file_manager'), + create_pattern('ui/gfx'), + create_pattern('ui/gl'), + create_pattern('ui/latency'), + create_pattern('ui/login'), + create_pattern('ui/message_center'), + create_pattern('ui/native_theme'), + create_pattern('ui/ozone'), + create_pattern('ui/platform_window'), + create_pattern('ui/resources'), + create_pattern('ui/shell_dialogs'), + create_pattern('ui/snapshot'), + create_pattern('ui/strings'), + create_pattern('ui/surface'), + create_pattern('ui/touch_selection'), + create_pattern('ui/views'), + create_pattern('ui/views_bridge_mac'), + create_pattern('ui/views_content_client'), + create_pattern('ui/web_dialogs'), + create_pattern('ui/webui'), + create_pattern('ui/wm'), + create_pattern('url'), + create_pattern('v8/benchmarks'), + create_pattern('v8/build_overrides'), + create_pattern('v8/custom_deps'), + create_pattern('v8/docs'), + create_pattern('v8/gni'), + create_pattern('v8/include'), + create_pattern('v8/infra'), + create_pattern('v8/samples'), + create_pattern('v8/src'), + create_pattern('v8/test'), + create_pattern('v8/testing'), + create_pattern('v8/third_party'), + create_pattern('v8/tools'), + + # keep out/obj and other patterns at the end. + [ + 'out/obj', '.*/(gen|obj[^/]*)/(include|EXECUTABLES|SHARED_LIBRARIES|' + 'STATIC_LIBRARIES|NATIVE_TESTS)/.*: warning:' + ], + ['other', '.*'] # all other unrecognized patterns +] diff --git a/tools/warn/severity.py b/tools/warn/severity.py index b1c38e4a3..b4c03c9ef 100644 --- a/tools/warn/severity.py +++ b/tools/warn/severity.py @@ -19,25 +19,27 @@ This file stores definition for class Severity that is used in warn_patterns. """ +# pylint:disable=old-style-class +class SeverityInfo: + + def __init__(self, value, color, column_header, header): + self.value = value + self.color = color + self.column_header = column_header + self.header = header + + # pylint:disable=old-style-class class Severity: """Class of Severity levels where each level is a SeverityInfo.""" - class SeverityInfo: - - def __init__(self, value, color, column_header, header): - self.value = value - self.color = color - self.column_header = column_header - self.header = header - # SEVERITY_UNKNOWN should never occur since every warn_pattern listed has # a specified severity. It exists for protobuf, the other values must # map to non-zero values (since 0 is reserved for a default UNKNOWN), but # logic in clang_tidy_warn.py assumes severity level values are consecutive # ints starting with 0. - SEVERITY_UNKNOWN = SeverityInfo(0, 'blueviolet', 'Errors of unknown severity', - 'Unknown severity (should not occur)') + SEVERITY_UNKNOWN = SeverityInfo(0, 'blueviolet', 'Unknown', + 'Unknown-severity warnings)') FIXMENOW = SeverityInfo(1, 'fuschia', 'FixNow', 'Critical warnings, fix me now') HIGH = SeverityInfo(2, 'red', 'High', 'High severity warnings') diff --git a/tools/warn/warn.py b/tools/warn/warn.py index bdfd48984..56e8787d9 100755 --- a/tools/warn/warn.py +++ b/tools/warn/warn.py @@ -17,21 +17,51 @@ """Simple wrapper to run warn_common with Python standard Pool.""" import multiprocessing +import signal +import sys # pylint:disable=relative-beyond-top-level -# pylint:disable=g-importing-member -from .warn_common import common_main +from . import warn_common as common -# This parallel_process could be changed depending on platform -# and availability of multi-process library functions. -def parallel_process(num_cpu, classify_warnings, groups): +def classify_warnings(args): + """Classify a list of warning lines. + + Args: + args: dictionary { + 'group': list of (warning, link), + 'project_patterns': re.compile(project_list[p][1]), + 'warn_patterns': list of warn_pattern, + 'num_processes': number of processes being used for multiprocessing } + Returns: + results: a list of the classified warnings. + """ + results = [] + for line, link in args['group']: + common.classify_one_warning(line, link, results, args['project_patterns'], + args['warn_patterns']) + + # After the main work, ignore all other signals to a child process, + # to avoid bad warning/error messages from the exit clean-up process. + if args['num_processes'] > 1: + signal.signal(signal.SIGTERM, lambda *args: sys.exit(-signal.SIGTERM)) + return results + + +def create_and_launch_subprocesses(num_cpu, classify_warnings_fn, arg_groups, + group_results): pool = multiprocessing.Pool(num_cpu) - return pool.map(classify_warnings, groups) + for cpu in range(num_cpu): + proc_result = pool.map(classify_warnings_fn, arg_groups[cpu]) + if proc_result is not None: + group_results.append(proc_result) + return group_results def main(): - common_main(parallel_process) + use_google3 = False + common.common_main(use_google3, create_and_launch_subprocesses, + classify_warnings) if __name__ == '__main__': diff --git a/tools/warn/warn_common.py b/tools/warn/warn_common.py index 0c9d9efbb..329f1e552 100755 --- a/tools/warn/warn_common.py +++ b/tools/warn/warn_common.py @@ -18,6 +18,8 @@ Default is to output warnings in HTML tables grouped by warning severity. Use option --byproject to output tables grouped by source file projects. Use option --gencsv to output warning counts in CSV format. + +Default input file is build.log, which can be changed with the --log flag. """ # List of important data structures and functions in this script. @@ -36,13 +38,11 @@ Use option --gencsv to output warning counts in CSV format. # project_patterns[p] re.compile(project_list[p][1]) # project_names[p] project_list[p][0] # warning_messages array of each warning message, without source url +# warning_links array of each warning code search link; for 'chrome' # warning_records array of [idx to warn_patterns, # idx to project_names, -# idx to warning_messages] -# android_root -# platform_version -# target_product -# target_variant +# idx to warning_messages, +# idx to warning_links] # parse_input_file # # To emit html page of warning messages: @@ -61,8 +61,9 @@ Use option --gencsv to output warning counts in CSV format. # # New dynamic HTML page's static JavaScript data: # Some data are copied from Python to JavaScript, to generate HTML elements. -# FlagURL args.url -# FlagSeparator args.separator +# FlagPlatform flags.platform +# FlagURL flags.url, used by 'android' +# FlagSeparator flags.separator, used by 'android' # SeverityColors: list of colors for all severity levels # SeverityHeaders: list of headers for all severity levels # SeverityColumnHeaders: list of column_headers for all severity levels @@ -88,80 +89,67 @@ import io import multiprocessing import os import re -import signal import sys # pylint:disable=relative-beyond-top-level -from . import cpp_warn_patterns -from . import java_warn_patterns -from . import make_warn_patterns -from . import other_warn_patterns -from . import tidy_warn_patterns +from . import android_project_list +from . import chrome_project_list +from . import cpp_warn_patterns as cpp_patterns +from . import java_warn_patterns as java_patterns +from . import make_warn_patterns as make_patterns +from . import other_warn_patterns as other_patterns +from . import tidy_warn_patterns as tidy_patterns # pylint:disable=g-importing-member -from .android_project_list import project_list from .severity import Severity -parser = argparse.ArgumentParser(description='Convert a build log into HTML') -parser.add_argument('--csvpath', - help='Save CSV warning file to the passed absolute path', - default=None) -parser.add_argument('--gencsv', - help='Generate a CSV file with number of various warnings', - action='store_true', - default=False) -parser.add_argument('--byproject', - help='Separate warnings in HTML output by project names', - action='store_true', - default=False) -parser.add_argument('--url', - help='Root URL of an Android source code tree prefixed ' - 'before files in warnings') -parser.add_argument('--separator', - help='Separator between the end of a URL and the line ' - 'number argument. e.g. #') -parser.add_argument('--processes', - type=int, - default=multiprocessing.cpu_count(), - help='Number of parallel processes to process warnings') -parser.add_argument(dest='buildlog', metavar='build.log', - help='Path to build.log file') -args = parser.parse_args() -warn_patterns = make_warn_patterns.warn_patterns -warn_patterns.extend(cpp_warn_patterns.warn_patterns) -warn_patterns.extend(java_warn_patterns.warn_patterns) -warn_patterns.extend(tidy_warn_patterns.warn_patterns) -warn_patterns.extend(other_warn_patterns.warn_patterns) - -project_patterns = [] -project_names = [] -warning_messages = [] -warning_records = [] +def parse_args(use_google3): + """Define and parse the args. Return the parse_args() result.""" + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--capacitor_path', default='', + help='Save capacitor warning file to the passed absolute' + ' path') + # csvpath has a different naming than the above path because historically the + # original Android script used csvpath, so other scripts rely on it + parser.add_argument('--csvpath', default='', + help='Save CSV warning file to the passed path') + parser.add_argument('--gencsv', action='store_true', + help='Generate CSV file with number of various warnings') + parser.add_argument('--byproject', action='store_true', + help='Separate warnings in HTML output by project names') + parser.add_argument('--url', default='', + help='Root URL of an Android source code tree prefixed ' + 'before files in warnings') + parser.add_argument('--separator', default='?l=', + help='Separator between the end of a URL and the line ' + 'number argument. e.g. #') + parser.add_argument('--processes', default=multiprocessing.cpu_count(), + type=int, + help='Number of parallel processes to process warnings') + # Old Android build scripts call warn.py without --platform, + # so the default platform is set to 'android'. + parser.add_argument('--platform', default='android', + choices=['chrome', 'android'], + help='Platform of the build log') + # Old Android build scripts call warn.py with only a build.log file path. + parser.add_argument('--log', help='Path to build log file') + parser.add_argument(dest='buildlog', metavar='build.log', + default='build.log', nargs='?', + help='Path to build.log file') + flags = parser.parse_args() + if not flags.log: + flags.log = flags.buildlog + if not use_google3 and not os.path.exists(flags.log): + sys.exit('Cannot find log file: ' + flags.log) + return flags -def initialize_arrays(): - """Complete global arrays before they are used.""" - global project_names, project_patterns - project_names = [p[0] for p in project_list] - project_patterns = [re.compile(p[1]) for p in project_list] - for w in warn_patterns: - w['members'] = [] - # Each warning pattern has a 'projects' dictionary, that - # maps a project name to number of warnings in that project. - w['projects'] = {} +def get_project_names(project_list): + """Get project_names from project_list.""" + return [p[0] for p in project_list] -initialize_arrays() - - -android_root = '' -platform_version = 'unknown' -target_product = 'unknown' -target_variant = 'unknown' - - -##### Data and functions to dump html file. ################################## - html_head_scripts = """\ ') -def dump_stats(writer): +def emit_stats_by_project(writer, warn_patterns, project_names): + """Dump a google chart table of warnings per project and severity.""" + + warnings = create_warnings(warn_patterns, project_names) + total_by_project = get_total_by_project(warnings, project_names) + total_by_severity = get_total_by_severity(warnings, project_names) + stats_header = emit_table_header(total_by_severity) + total_all_projects, stats_rows = \ + emit_row_counts_per_project(warnings, total_by_project, total_by_severity, project_names) + emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows, + total_all_projects, writer) + + +def dump_stats(writer, warn_patterns): """Dump some stats about total number of warnings and such.""" + known = 0 skipped = 0 unknown = 0 - sort_warnings() + sort_warnings(warn_patterns) for i in warn_patterns: if i['severity'] == Severity.UNMATCHED: unknown += len(i['members']) @@ -344,7 +394,7 @@ def all_patterns(category): return patterns -def dump_fixed(writer): +def dump_fixed(writer, warn_patterns): """Show which warnings no longer occur.""" anchor = 'fixed_warnings' mark = anchor + '_mark' @@ -372,95 +422,156 @@ def dump_fixed(writer): writer('') -def find_project_index(line): - for p in range(len(project_patterns)): - if project_patterns[p].match(line): - return p +def write_severity(csvwriter, sev, kind, warn_patterns): + """Count warnings of given severity and write CSV entries to writer.""" + total = 0 + for pattern in warn_patterns: + if pattern['severity'] == sev and pattern['members']: + n = len(pattern['members']) + total += n + warning = kind + ': ' + (pattern['description'] or '?') + csvwriter.writerow([n, '', warning]) + # print number of warnings for each project, ordered by project name + projects = sorted(pattern['projects'].keys()) + for project in projects: + csvwriter.writerow([pattern['projects'][project], project, warning]) + csvwriter.writerow([total, '', kind + ' warnings']) + return total + + +def dump_csv(csvwriter, warn_patterns): + """Dump number of warnings in CSV format to writer.""" + sort_warnings(warn_patterns) + total = 0 + for s in Severity.levels: + total += write_severity(csvwriter, s, s.column_header, warn_patterns) + csvwriter.writerow([total, '', 'All warnings']) + + +def find_project_index(line, project_patterns): + for i, p in enumerate(project_patterns): + if p.match(line): + return i return -1 -def classify_one_warning(line, results): +def classify_one_warning(warning, link, results, project_patterns, + warn_patterns): """Classify one warning line.""" - for i in range(len(warn_patterns)): - w = warn_patterns[i] + for i, w in enumerate(warn_patterns): for cpat in w['compiled_patterns']: - # pytype: disable=attribute-error - if cpat.match(line): - p = find_project_index(line) - results.append([line, i, p]) + if cpat.match(warning): + p = find_project_index(warning, project_patterns) + results.append([warning, link, i, p]) return else: # If we end up here, there was a problem parsing the log # probably caused by 'make -j' mixing the output from # 2 or more concurrent compiles pass - # pytype: enable=attribute-error -def classify_warnings(lines): - results = [] - for line in lines: - classify_one_warning(line, results) - # After the main work, ignore all other signals to a child process, - # to avoid bad warning/error messages from the exit clean-up process. - if args.processes > 1: - signal.signal(signal.SIGTERM, lambda *args: sys.exit(-signal.SIGTERM)) - return results +def remove_prefix(s, sub): + """Remove everything before last occurrence of substring sub in string s.""" + if sub in s: + inc_sub = s.rfind(sub) + return s[inc_sub:] + return s -def parallel_classify_warnings(warning_lines, parallel_process): - """Classify all warning lines with num_cpu parallel processes.""" - num_cpu = args.processes - if num_cpu > 1: - groups = [[] for x in range(num_cpu)] - i = 0 - for x in warning_lines: - groups[i].append(x) - i = (i + 1) % num_cpu - group_results = parallel_process(num_cpu, classify_warnings, groups) - else: - group_results = [classify_warnings(warning_lines)] +# TODO(emmavukelj): Don't have any generate_*_cs_link functions call +# normalize_path a second time (the first time being in parse_input_file) +def generate_cs_link(warning_line, flags, android_root=None): + if flags.platform == 'chrome': + return generate_chrome_cs_link(warning_line, flags) + if flags.platform == 'android': + return generate_android_cs_link(warning_line, flags, android_root) + return 'https://cs.corp.google.com/' - for result in group_results: - for line, pattern_idx, project_idx in result: - pattern = warn_patterns[pattern_idx] - pattern['members'].append(line) - message_idx = len(warning_messages) - warning_messages.append(line) - warning_records.append([pattern_idx, project_idx, message_idx]) - pname = '???' if project_idx < 0 else project_names[project_idx] - # Count warnings by project. - if pname in pattern['projects']: - pattern['projects'][pname] += 1 - else: - pattern['projects'][pname] = 1 + +def generate_android_cs_link(warning_line, flags, android_root): + """Generate the code search link for a warning line in Android.""" + # max_splits=2 -> only 3 items + raw_path, line_number_str, _ = warning_line.split(':', 2) + normalized_path = normalize_path(raw_path, flags, android_root) + if not flags.url: + return normalized_path + link_path = flags.url + '/' + normalized_path + if line_number_str.isdigit(): + link_path += flags.separator + line_number_str + return link_path + + +def generate_chrome_cs_link(warning_line, flags): + """Generate the code search link for a warning line in Chrome.""" + split_line = warning_line.split(':') + raw_path = split_line[0] + normalized_path = normalize_path(raw_path, flags) + link_base = 'https://cs.chromium.org/' + link_add = 'chromium' + link_path = None + + # Basically just going through a few specific directory cases and specifying + # the proper behavior for that case. This list of cases was accumulated + # through trial and error manually going through the warnings. + # + # This code pattern of using case-specific "if"s instead of "elif"s looks + # possibly accidental and mistaken but it is intentional because some paths + # fall under several cases (e.g. third_party/lib/nghttp2_frame.c) and for + # those we want the most specific case to be applied. If there is reliable + # knowledge of exactly where these occur, this could be changed to "elif"s + # but there is no reliable set of paths falling under multiple cases at the + # moment. + if '/src/third_party' in raw_path: + link_path = remove_prefix(raw_path, '/src/third_party/') + if '/chrome_root/src_internal/' in raw_path: + link_path = remove_prefix(raw_path, '/chrome_root/src_internal/') + link_path = link_path[len('/chrome_root'):] # remove chrome_root + if '/chrome_root/src/' in raw_path: + link_path = remove_prefix(raw_path, '/chrome_root/src/') + link_path = link_path[len('/chrome_root'):] # remove chrome_root + if '/libassistant/' in raw_path: + link_add = 'eureka_internal/chromium/src' + link_base = 'https://cs.corp.google.com/' # internal data + link_path = remove_prefix(normalized_path, '/libassistant/') + if raw_path.startswith('gen/'): + link_path = '/src/out/Debug/gen/' + normalized_path + if '/gen/' in raw_path: + return '%s?q=file:%s' % (link_base, remove_prefix(normalized_path, '/gen/')) + + if not link_path and (raw_path.startswith('src/') or + raw_path.startswith('src_internal/')): + link_path = '/%s' % raw_path + + if not link_path: # can't find specific link, send a query + return '%s?q=file:%s' % (link_base, normalized_path) + + line_number = int(split_line[1]) + link = '%s%s%s?l=%d' % (link_base, link_add, link_path, line_number) + return link def find_warn_py_and_android_root(path): - """Set and return android_root path if it is found.""" - global android_root + """Return android source root path if warn.py is found.""" parts = path.split('/') for idx in reversed(range(2, len(parts))): root_path = '/'.join(parts[:idx]) # Android root directory should contain this script. if os.path.exists(root_path + '/build/make/tools/warn.py'): - android_root = root_path - return True - return False + return root_path + return '' -def find_android_root(): - """Guess android_root from common prefix of file paths.""" +def find_android_root(buildlog): + """Guess android source root from common prefix of file paths.""" # Use the longest common prefix of the absolute file paths # of the first 10000 warning messages as the android_root. - global android_root - warning_lines = set() + warning_lines = [] warning_pattern = re.compile('^/[^ ]*/[^ ]*: warning: .*') count = 0 - infile = io.open(args.buildlog, mode='r', encoding='utf-8') - for line in infile: + for line in buildlog: if warning_pattern.match(line): - warning_lines.add(line) + warning_lines.append(line) count += 1 if count > 9999: break @@ -468,56 +579,110 @@ def find_android_root(): # the source tree root. if count < 100: path = os.path.normpath(re.sub(':.*$', '', line)) - if find_warn_py_and_android_root(path): - return + android_root = find_warn_py_and_android_root(path) + if android_root: + return android_root # Do not use common prefix of a small number of paths. if count > 10: # pytype: disable=wrong-arg-types root_path = os.path.commonprefix(warning_lines) # pytype: enable=wrong-arg-types if len(root_path) > 2 and root_path[len(root_path) - 1] == '/': - android_root = root_path[:-1] + return root_path[:-1] + return '' -def remove_android_root_prefix(path): +def remove_android_root_prefix(path, android_root): """Remove android_root prefix from path if it is found.""" if path.startswith(android_root): return path[1 + len(android_root):] + return path + + +def normalize_path(path, flags, android_root=None): + """Normalize file path relative to src/ or src-internal/ directory.""" + path = os.path.normpath(path) + + if flags.platform == 'android': + if android_root: + return remove_android_root_prefix(path, android_root) + return path + + # Remove known prefix of root path and normalize the suffix. + idx = path.find('chrome_root/') + if idx >= 0: + # remove chrome_root/, we want path relative to that + return path[idx + len('chrome_root/'):] else: return path -def normalize_path(path): - """Normalize file path relative to android_root.""" - # If path is not an absolute path, just normalize it. - path = os.path.normpath(path) - # Remove known prefix of root path and normalize the suffix. - if path[0] == '/' and android_root: - return remove_android_root_prefix(path) - return path - - -def normalize_warning_line(line): - """Normalize file path relative to android_root in a warning line.""" - # replace fancy quotes with plain ol' quotes +def normalize_warning_line(line, flags, android_root=None): + """Normalize file path relative to src directory in a warning line.""" line = re.sub(u'[\u2018\u2019]', '\'', line) # replace non-ASCII chars to spaces line = re.sub(u'[^\x00-\x7f]', ' ', line) line = line.strip() first_column = line.find(':') - if first_column > 0: - return normalize_path(line[:first_column]) + line[first_column:] - else: - return line + return normalize_path(line[:first_column], flags, + android_root) + line[first_column:] -def parse_input_file(infile): - """Parse input file, collect parameters and warning lines.""" - global android_root - global platform_version - global target_product - global target_variant - line_counter = 0 +def parse_input_file_chrome(infile, flags): + """Parse Chrome input file, collect parameters and warning lines.""" + platform_version = 'unknown' + board_name = 'unknown' + architecture = 'unknown' + + # only handle warning lines of format 'file_path:line_no:col_no: warning: ...' + chrome_warning_pattern = r'^[^ ]*/[^ ]*:[0-9]+:[0-9]+: warning: .*' + + warning_pattern = re.compile(chrome_warning_pattern) + + # Collect all unique warning lines + # Remove the duplicated warnings save ~8% of time when parsing + # one typical build log than before + unique_warnings = dict() + for line in infile: + if warning_pattern.match(line): + normalized_line = normalize_warning_line(line, flags) + if normalized_line not in unique_warnings: + unique_warnings[normalized_line] = generate_cs_link(line, flags) + elif (platform_version == 'unknown' or board_name == 'unknown' or + architecture == 'unknown'): + m = re.match(r'.+Package:.+chromeos-base/chromeos-chrome-', line) + if m is not None: + platform_version = 'R' + line.split('chrome-')[1].split('_')[0] + continue + m = re.match(r'.+Source\sunpacked\sin\s(.+)', line) + if m is not None: + board_name = m.group(1).split('/')[2] + continue + m = re.match(r'.+USE:\s*([^\s]*).*', line) + if m is not None: + architecture = m.group(1) + continue + + header_str = '%s - %s - %s' % (platform_version, board_name, architecture) + return unique_warnings, header_str + + +def add_normalized_line_to_warnings(line, flags, android_root, unique_warnings): + """Parse/normalize path, updating warning line and add to warnings dict.""" + normalized_line = normalize_warning_line(line, flags, android_root) + if normalized_line not in unique_warnings: + unique_warnings[normalized_line] = generate_cs_link(line, flags, + android_root) + return unique_warnings + + +def parse_input_file_android(infile, flags): + """Parse Android input file, collect parameters and warning lines.""" + platform_version = 'unknown' + target_product = 'unknown' + target_variant = 'unknown' + android_root = find_android_root(infile) + infile.seek(0) # rustc warning messages have two lines that should be combined: # warning: description @@ -532,20 +697,25 @@ def parse_input_file(infile): warning_without_file = re.compile('^warning: .*') rustc_file_position = re.compile('^[ ]+--> [^ ]*/[^ ]*:[0-9]+:[0-9]+') - # Collect all warnings into the warning_lines set. - warning_lines = set() + # Collect all unique warning lines + # Remove the duplicated warnings save ~8% of time when parsing + # one typical build log than before + unique_warnings = dict() + line_counter = 0 prev_warning = '' for line in infile: if prev_warning: if rustc_file_position.match(line): # must be a rustc warning, combine 2 lines into one warning line = line.strip().replace('--> ', '') + ': ' + prev_warning - warning_lines.add(normalize_warning_line(line)) + unique_warnings = add_normalized_line_to_warnings( + line, flags, android_root, unique_warnings) prev_warning = '' continue # add prev_warning, and then process the current line prev_warning = 'unknown_source_file: ' + prev_warning - warning_lines.add(normalize_warning_line(prev_warning)) + unique_warnings = add_normalized_line_to_warnings( + prev_warning, flags, android_root, unique_warnings) prev_warning = '' if warning_pattern.match(line): @@ -553,7 +723,8 @@ def parse_input_file(infile): # save this line and combine it with the next line prev_warning = line else: - warning_lines.add(normalize_warning_line(line)) + unique_warnings = add_normalized_line_to_warnings( + line, flags, android_root, unique_warnings) continue if line_counter < 100: @@ -568,17 +739,36 @@ def parse_input_file(infile): m = re.search('(?<=^TARGET_BUILD_VARIANT=).*', line) if m is not None: target_variant = m.group(0) - m = re.search('.* TOP=([^ ]*) .*', line) + m = re.search('(?<=^TOP=).*', line) if m is not None: android_root = m.group(1) - return warning_lines + + if android_root: + new_unique_warnings = dict() + for warning_line in unique_warnings: + normalized_line = normalize_warning_line(warning_line, flags, + android_root) + new_unique_warnings[normalized_line] = generate_android_cs_link( + warning_line, flags, android_root) + unique_warnings = new_unique_warnings + + header_str = '%s - %s - %s' % (platform_version, target_product, + target_variant) + return unique_warnings, header_str + + +def parse_input_file(infile, flags): + if flags.platform == 'chrome': + return parse_input_file_chrome(infile, flags) + if flags.platform == 'android': + return parse_input_file_android(infile, flags) + raise RuntimeError('parse_input_file not defined for platform %s' % + flags.platform) # Return s with escaped backslash and quotation characters. def escape_string(s): - # pytype: disable=attribute-error return s.replace('\\', '\\\\').replace('"', '\\"') - # pytype: enable=attribute-error # Return s without trailing '\n' and escape the quotation characters. @@ -589,19 +779,22 @@ def strip_escape_string(s): return escape_string(s) -def emit_warning_array(name, writer): +def emit_warning_array(name, writer, warn_patterns): writer('var warning_{} = ['.format(name)) - for i in range(len(warn_patterns)): - writer('{},'.format(warn_patterns[i][name])) + for w in warn_patterns: + if name == 'severity': + writer('{},'.format(w[name].value)) + else: + writer('{},'.format(w[name])) writer('];') -def emit_warning_arrays(writer): - emit_warning_array('severity', writer) +def emit_warning_arrays(writer, warn_patterns): + emit_warning_array('severity', writer, warn_patterns) writer('var warning_description = [') - for i in range(len(warn_patterns)): - if warn_patterns[i]['members']: - writer('"{}",'.format(escape_string(warn_patterns[i]['description']))) + for w in warn_patterns: + if w['members']: + writer('"{}",'.format(escape_string(w['description']))) else: writer('"",') # no such warning writer('];') @@ -621,7 +814,7 @@ scripts_for_warning_groups = """ return byMessageCount(x1, x2); } const ParseLinePattern = /^([^ :]+):(\\d+):(.+)/; - function addURL(line) { + function addURL(line) { // used by Android if (FlagURL == "") return line; if (FlagSeparator == "") { return line.replace(ParseLinePattern, @@ -631,6 +824,13 @@ scripts_for_warning_groups = """ "$1:$2:$3"); } + function addURLToLine(line, link) { // used by Chrome + let line_split = line.split(":"); + let path = line_split.slice(0,3).join(":"); + let msg = line_split.slice(3).join(":"); + let html_link = `${path}${msg}`; + return html_link; + } function createArrayOfDictionaries(n) { var result = []; for (var i=0; i"; var c = 0; messages.sort(compareMessages); - for (var i=0; i" + - addURL(WarningMessages[messages[i][2]]) + ""; - c = 1 - c; + if (FlagPlatform == "chrome") { + for (var i=0; i" + + addURLToLine(WarningMessages[messages[i][2]], WarningLinks[messages[i][3]]) + ""; + c = 1 - c; + } + } else { + for (var i=0; i" + + addURL(WarningMessages[messages[i][2]]) + ""; + c = 1 - c; + } } result += "
"; } @@ -777,14 +985,14 @@ def emit_const_object_array(name, array, writer): writer('];') -def emit_js_data(writer): +def emit_js_data(writer, flags, warning_messages, warning_links, + warning_records, warn_patterns, project_names): """Dump dynamic HTML page's static JavaScript data.""" - emit_const_string('FlagURL', - args.url if args.url else '', writer) - emit_const_string('FlagSeparator', - args.separator if args.separator else '', writer) - emit_const_string_array('SeverityColors', - [s.color for s in Severity.levels], writer) + emit_const_string('FlagPlatform', flags.platform, writer) + emit_const_string('FlagURL', flags.url, writer) + emit_const_string('FlagSeparator', flags.separator, writer) + emit_const_string_array('SeverityColors', [s.color for s in Severity.levels], + writer) emit_const_string_array('SeverityHeaders', [s.header for s in Severity.levels], writer) emit_const_string_array('SeverityColumnHeaders', @@ -799,6 +1007,9 @@ def emit_js_data(writer): writer) emit_const_html_string_array('WarningMessages', warning_messages, writer) emit_const_object_array('Warnings', warning_records, writer) + if flags.platform == 'chrome': + emit_const_html_string_array('WarningLinks', warning_links, writer) + draw_table_javascript = """ google.charts.load('current', {'packages':['table']}); @@ -822,82 +1033,201 @@ function drawTable() { """ -def dump_html(output_stream): - """Dump the html output to output_stream.""" +def dump_html(flags, output_stream, warning_messages, warning_links, + warning_records, header_str, warn_patterns, project_names): + """Dump the flags output to output_stream.""" writer = make_writer(output_stream) - dump_html_prologue('Warnings for ' + platform_version + ' - ' + - target_product + ' - ' + target_variant, writer) - dump_stats(writer) + dump_html_prologue('Warnings for ' + header_str, writer, warn_patterns, + project_names) + dump_stats(writer, warn_patterns) writer('

') writer('\n') emit_buttons(writer) # Warning messages are grouped by severities or project names. writer('
') - if args.byproject: + if flags.byproject: writer('') else: writer('') - dump_fixed(writer) + dump_fixed(writer, warn_patterns) dump_html_epilogue(writer) -##### Functions to count warnings and dump csv file. ######################### +def parse_compiler_output(compiler_output): + """Parse compiler output for relevant info.""" + split_output = compiler_output.split(':', 3) # 3 = max splits + file_path = split_output[0] + line_number = int(split_output[1]) + col_number = int(split_output[2].split(' ')[0]) + warning_message = split_output[3] + return file_path, line_number, col_number, warning_message -def description_for_csv(category): - if not category['description']: - return '?' - return category['description'] - - -def count_severity(writer, sev, kind): - """Count warnings of given severity.""" - total = 0 - for i in warn_patterns: - if i['severity'] == sev and i['members']: - n = len(i['members']) - total += n - warning = kind + ': ' + description_for_csv(i) - writer.writerow([n, '', warning]) - # print number of warnings for each project, ordered by project name. - # pytype: disable=attribute-error - projects = sorted(i['projects'].keys()) - # pytype: enable=attribute-error - for p in projects: - writer.writerow([i['projects'][p], p, warning]) - writer.writerow([total, '', kind + ' warnings']) - - return total - - -# dump number of warnings in csv format to stdout -def dump_csv(writer): - """Dump number of warnings in csv format to stdout.""" - sort_warnings() - total = 0 - for s in Severity.levels: - if s != Severity.SEVERITY_UNKNOWN: - total += count_severity(writer, s, s.column_header) - writer.writerow([total, '', 'All warnings']) - - -def common_main(parallel_process): - """Real main function to classify warnings and generate .html file.""" - find_android_root() - # We must use 'utf-8' codec to parse some non-ASCII code in warnings. - warning_lines = parse_input_file( - io.open(args.buildlog, mode='r', encoding='utf-8')) - parallel_classify_warnings(warning_lines, parallel_process) - # If a user pases a csv path, save the fileoutput to the path - # If the user also passed gencsv write the output to stdout - # If the user did not pass gencsv flag dump the html report to stdout. - if args.csvpath: - with open(args.csvpath, 'w') as f: - dump_csv(csv.writer(f, lineterminator='\n')) - if args.gencsv: - dump_csv(csv.writer(sys.stdout, lineterminator='\n')) +def get_warn_patterns(platform): + """Get and initialize warn_patterns.""" + warn_patterns = [] + if platform == 'chrome': + warn_patterns = cpp_patterns.warn_patterns + elif platform == 'android': + warn_patterns = make_patterns.warn_patterns + cpp_patterns.warn_patterns + java_patterns.warn_patterns + tidy_patterns.warn_patterns + other_patterns.warn_patterns else: - dump_html(sys.stdout) + raise Exception('platform name %s is not valid' % platform) + for w in warn_patterns: + w['members'] = [] + # Each warning pattern has a 'projects' dictionary, that + # maps a project name to number of warnings in that project. + w['projects'] = {} + return warn_patterns + + +def get_project_list(platform): + """Return project list for appropriate platform.""" + if platform == 'chrome': + return chrome_project_list.project_list + if platform == 'android': + return android_project_list.project_list + raise Exception('platform name %s is not valid' % platform) + + +def parallel_classify_warnings(warning_data, args, project_names, + project_patterns, warn_patterns, + use_google3, create_launch_subprocs_fn, + classify_warnings_fn): + """Classify all warning lines with num_cpu parallel processes.""" + num_cpu = args.processes + group_results = [] + + if num_cpu > 1: + # set up parallel processing for this... + warning_groups = [[] for _ in range(num_cpu)] + i = 0 + for warning, link in warning_data.items(): + warning_groups[i].append((warning, link)) + i = (i + 1) % num_cpu + arg_groups = [[] for _ in range(num_cpu)] + for i, group in enumerate(warning_groups): + arg_groups[i] = [{ + 'group': group, + 'project_patterns': project_patterns, + 'warn_patterns': warn_patterns, + 'num_processes': num_cpu + }] + + group_results = create_launch_subprocs_fn(num_cpu, + classify_warnings_fn, + arg_groups, + group_results) + else: + group_results = [] + for warning, link in warning_data.items(): + classify_one_warning(warning, link, group_results, + project_patterns, warn_patterns) + group_results = [group_results] + + warning_messages = [] + warning_links = [] + warning_records = [] + if use_google3: + group_results = [group_results] + for group_result in group_results: + for result in group_result: + for line, link, pattern_idx, project_idx in result: + pattern = warn_patterns[pattern_idx] + pattern['members'].append(line) + message_idx = len(warning_messages) + warning_messages.append(line) + link_idx = len(warning_links) + warning_links.append(link) + warning_records.append([pattern_idx, project_idx, message_idx, + link_idx]) + pname = '???' if project_idx < 0 else project_names[project_idx] + # Count warnings by project. + if pname in pattern['projects']: + pattern['projects'][pname] += 1 + else: + pattern['projects'][pname] = 1 + return warning_messages, warning_links, warning_records + + +def write_html(flags, project_names, warn_patterns, html_path, warning_messages, + warning_links, warning_records, header_str): + """Write warnings html file.""" + if html_path: + with open(html_path, 'w') as f: + dump_html(flags, f, warning_messages, warning_links, warning_records, + header_str, warn_patterns, project_names) + + +def write_out_csv(flags, warn_patterns, warning_messages, warning_links, + warning_records, header_str, project_names): + """Write warnings csv file.""" + if flags.csvpath: + with open(flags.csvpath, 'w') as f: + dump_csv(csv.writer(f, lineterminator='\n'), warn_patterns) + + if flags.gencsv: + dump_csv(csv.writer(sys.stdout, lineterminator='\n'), warn_patterns) + else: + dump_html(flags, sys.stdout, warning_messages, warning_links, + warning_records, header_str, warn_patterns, project_names) + + +def process_log(logfile, flags, project_names, project_patterns, warn_patterns, + html_path, use_google3, create_launch_subprocs_fn, + classify_warnings_fn, logfile_object): + # pylint: disable=g-doc-args + # pylint: disable=g-doc-return-or-yield + """Function that handles processing of a log. + + This is isolated into its own function (rather than just taking place in main) + so that it can be used by both warn.py and the borg job process_gs_logs.py, to + avoid duplication of code. + Note that if the arguments to this function change, process_gs_logs.py must + be updated accordingly. + """ + if logfile_object is None: + with io.open(logfile, encoding='utf-8') as log: + warning_lines_and_links, header_str = parse_input_file(log, flags) + else: + warning_lines_and_links, header_str = parse_input_file( + logfile_object, flags) + warning_messages, warning_links, warning_records = parallel_classify_warnings( + warning_lines_and_links, flags, project_names, project_patterns, + warn_patterns, use_google3, create_launch_subprocs_fn, + classify_warnings_fn) + + write_html(flags, project_names, warn_patterns, html_path, + warning_messages, warning_links, warning_records, + header_str) + + return warning_messages, warning_links, warning_records, header_str + + +def common_main(use_google3, create_launch_subprocs_fn, classify_warnings_fn, + logfile_object=None): + """Shared main function for Google3 and non-Google3 versions of warn.py.""" + flags = parse_args(use_google3) + warn_patterns = get_warn_patterns(flags.platform) + project_list = get_project_list(flags.platform) + + project_names = get_project_names(project_list) + project_patterns = [re.compile(p[1]) for p in project_list] + + # html_path=None because we output html below if not outputting CSV + warning_messages, warning_links, warning_records, header_str = process_log( + logfile=flags.log, flags=flags, project_names=project_names, + project_patterns=project_patterns, warn_patterns=warn_patterns, + html_path=None, use_google3=use_google3, + create_launch_subprocs_fn=create_launch_subprocs_fn, + classify_warnings_fn=classify_warnings_fn, + logfile_object=logfile_object) + + write_out_csv(flags, warn_patterns, warning_messages, warning_links, + warning_records, header_str, project_names) + + # Return these values, so that caller can use them, if desired. + return flags, warning_messages, warning_records, warn_patterns