forked from openkylin/platform_build
Use .KATI_RESTAT to reduce unnecessary rebuilds of .jar files
The same optimization was done for binaries in
https://android-review.googlesource.com/#/c/175250/
To create a TOC file from .jar files, this change introduces
ijar, which is designed for this purpose. Only #include lines
were modified from the original version.
https://github.com/bazelbuild/bazel/tree/master/third_party/ijar
Performance:
$ m && touch
frameworks/base/core/java/com/google/android/util/Procedure.java && time
m
Before: 4m30s (1580 targets)
After: 3m57s (772 targets)
Unfortunately, the improvement is small yet, but local
experiments showed we can cut ~2 more minutes if the similar
optimization is done for .dex files.
(cherry picked from commit c1f5d9c203)
Bug: 24597504
Change-Id: Iec3b2b0b0e674bee5d80cce3c300dc8fad6e7c13
This commit is contained in:
Shinichiro Hamaji
Colin Cross
parent
2ff3ad9045
commit
89b255ab71
|
|
@ -223,25 +223,15 @@ LOCAL_INTERMEDIATE_TARGETS += $(LOCAL_BUILT_MODULE)
|
||||||
###########################################################
|
###########################################################
|
||||||
## Create .toc files from shared objects to reduce unnecessary rebuild
|
## Create .toc files from shared objects to reduce unnecessary rebuild
|
||||||
# .toc files have the list of external dynamic symbols without their addresses.
|
# .toc files have the list of external dynamic symbols without their addresses.
|
||||||
# For ninja build, .toc files will be updated only when the content of .toc
|
# As .KATI_RESTAT is specified to .toc files and commit-change-for-toc is used,
|
||||||
# files are changed. As .KATI_RESTAT is specified to .toc files, dependent
|
# dependent binaries of a .toc file will be rebuilt only when the content of
|
||||||
# binaries of a .toc file will be rebuilt only when the content of
|
|
||||||
# the .toc file is changed.
|
# the .toc file is changed.
|
||||||
###########################################################
|
###########################################################
|
||||||
ifeq ($(LOCAL_MODULE_CLASS),SHARED_LIBRARIES)
|
ifeq ($(LOCAL_MODULE_CLASS),SHARED_LIBRARIES)
|
||||||
LOCAL_INTERMEDIATE_TARGETS += $(LOCAL_BUILT_MODULE).toc
|
LOCAL_INTERMEDIATE_TARGETS += $(LOCAL_BUILT_MODULE).toc
|
||||||
$(LOCAL_BUILT_MODULE).toc: $(LOCAL_BUILT_MODULE)
|
$(LOCAL_BUILT_MODULE).toc: $(LOCAL_BUILT_MODULE)
|
||||||
ifeq ($(BUILDING_WITH_NINJA),true)
|
|
||||||
$(call $(PRIVATE_2ND_ARCH_VAR_PREFIX)$(PRIVATE_PREFIX)transform-shared-lib-to-toc,$<,$@.tmp)
|
$(call $(PRIVATE_2ND_ARCH_VAR_PREFIX)$(PRIVATE_PREFIX)transform-shared-lib-to-toc,$<,$@.tmp)
|
||||||
$(hide) if cmp -s $@.tmp $@ ; then \
|
$(call commit-change-for-toc,$@)
|
||||||
rm $@.tmp ; \
|
|
||||||
else \
|
|
||||||
mv $@.tmp $@ ; \
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
@# make doesn't support restat. We always update .toc files so the dependents will always be updated too.
|
|
||||||
$(call $(PRIVATE_2ND_ARCH_VAR_PREFIX)$(PRIVATE_PREFIX)transform-shared-lib-to-toc,$<,$@)
|
|
||||||
endif
|
|
||||||
|
|
||||||
# Kati adds restat=1 to ninja. GNU make does nothing for this.
|
# Kati adds restat=1 to ninja. GNU make does nothing for this.
|
||||||
.KATI_RESTAT: $(LOCAL_BUILT_MODULE).toc
|
.KATI_RESTAT: $(LOCAL_BUILT_MODULE).toc
|
||||||
|
|
|
||||||
|
|
@ -540,6 +540,10 @@ ifndef TARGET_BUILD_APPS
|
||||||
ZIPTIME := $(HOST_OUT_EXECUTABLES)/ziptime$(HOST_EXECUTABLE_SUFFIX)
|
ZIPTIME := $(HOST_OUT_EXECUTABLES)/ziptime$(HOST_EXECUTABLE_SUFFIX)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
|
# ijar converts a .jar file to a smaller .jar file which only has its
|
||||||
|
# interfaces.
|
||||||
|
IJAR := $(HOST_OUT_EXECUTABLES)/ijar$(BUILD_EXECUTABLE_SUFFIX)
|
||||||
|
|
||||||
# relocation packer
|
# relocation packer
|
||||||
RELOCATION_PACKER := prebuilts/misc/$(BUILD_OS)-$(HOST_PREBUILT_ARCH)/relocation_packer/relocation_packer
|
RELOCATION_PACKER := prebuilts/misc/$(BUILD_OS)-$(HOST_PREBUILT_ARCH)/relocation_packer/relocation_packer
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1928,6 +1928,42 @@ define transform-jar-to-jack
|
||||||
$(hide) rm $@.tmpjill.jack
|
$(hide) rm $@.tmpjill.jack
|
||||||
endef
|
endef
|
||||||
|
|
||||||
|
# Moves $1.tmp to $1 if necessary. This is designed to be used with
|
||||||
|
# .KATI_RESTAT. For kati, this function doesn't update the timestamp
|
||||||
|
# of $1 when $1.tmp is identical to $1 so that ninja won't rebuild
|
||||||
|
# targets which depend on $1. For GNU make, this function simply
|
||||||
|
# copies $1.tmp to $1.
|
||||||
|
ifeq ($(BUILDING_WITH_NINJA),true)
|
||||||
|
define commit-change-for-toc
|
||||||
|
$(hide) if cmp -s $1.tmp $1 ; then \
|
||||||
|
rm $1.tmp ; \
|
||||||
|
else \
|
||||||
|
mv $1.tmp $1 ; \
|
||||||
|
fi
|
||||||
|
endef
|
||||||
|
else
|
||||||
|
define commit-change-for-toc
|
||||||
|
@# make doesn't support restat. We always update .toc files so the dependents will always be updated too.
|
||||||
|
$(hide) mv $1.tmp $1
|
||||||
|
endef
|
||||||
|
endif
|
||||||
|
|
||||||
|
## Rule to creates a table of contents from a .jar file.
|
||||||
|
## Must be called with $(eval).
|
||||||
|
# $1: A .jar file
|
||||||
|
define _transform-jar-to-toc
|
||||||
|
$1.toc: $1 | $(IJAR)
|
||||||
|
@echo Generating TOC: $$@
|
||||||
|
$(hide) $(IJAR) $$< $$@.tmp
|
||||||
|
$$(call commit-change-for-toc,$$@)
|
||||||
|
endef
|
||||||
|
|
||||||
|
## Define a rule which generates .jar.toc and mark it as .KATI_RESTAT.
|
||||||
|
define define-jar-to-toc-rule
|
||||||
|
$(eval $(call _transform-jar-to-toc,$1))
|
||||||
|
$(eval .KATI_RESTAT: $1.toc)
|
||||||
|
endef
|
||||||
|
|
||||||
|
|
||||||
# Invoke Jack to compile java from source to jack files without shrink or obfuscation.
|
# Invoke Jack to compile java from source to jack files without shrink or obfuscation.
|
||||||
#
|
#
|
||||||
|
|
|
||||||
|
|
@ -456,6 +456,8 @@ $(full_classes_jar): $(full_classes_emma_jar) | $(ACP)
|
||||||
@echo Copying: $@
|
@echo Copying: $@
|
||||||
$(hide) $(ACP) -fp $< $@
|
$(hide) $(ACP) -fp $< $@
|
||||||
|
|
||||||
|
$(call define-jar-to-toc-rule, $(full_classes_jar))
|
||||||
|
|
||||||
# Run proguard if necessary, otherwise just copy the file.
|
# Run proguard if necessary, otherwise just copy the file.
|
||||||
ifdef LOCAL_PROGUARD_ENABLED
|
ifdef LOCAL_PROGUARD_ENABLED
|
||||||
ifneq ($(filter-out full custom nosystem obfuscation optimization shrinktests,$(LOCAL_PROGUARD_ENABLED)),)
|
ifneq ($(filter-out full custom nosystem obfuscation optimization shrinktests,$(LOCAL_PROGUARD_ENABLED)),)
|
||||||
|
|
|
||||||
|
|
@ -147,6 +147,7 @@ endif # LOCAL_SDK_VERSION
|
||||||
|
|
||||||
full_shared_java_libs := $(call java-lib-files,$(LOCAL_JAVA_LIBRARIES),$(LOCAL_IS_HOST_MODULE))
|
full_shared_java_libs := $(call java-lib-files,$(LOCAL_JAVA_LIBRARIES),$(LOCAL_IS_HOST_MODULE))
|
||||||
full_java_lib_deps := $(call java-lib-deps,$(LOCAL_JAVA_LIBRARIES),$(LOCAL_IS_HOST_MODULE))
|
full_java_lib_deps := $(call java-lib-deps,$(LOCAL_JAVA_LIBRARIES),$(LOCAL_IS_HOST_MODULE))
|
||||||
|
full_java_lib_deps := $(addsuffix .toc, $(full_java_lib_deps))
|
||||||
|
|
||||||
else # LOCAL_IS_HOST_MODULE
|
else # LOCAL_IS_HOST_MODULE
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -315,6 +315,8 @@ $(common_classes_jar) : $(my_src_jar) | $(ACP)
|
||||||
$(common_javalib_jar) : $(common_classes_jar) | $(ACP)
|
$(common_javalib_jar) : $(common_classes_jar) | $(ACP)
|
||||||
$(transform-prebuilt-to-target)
|
$(transform-prebuilt-to-target)
|
||||||
|
|
||||||
|
$(call define-jar-to-toc-rule, $(common_classes_jar))
|
||||||
|
|
||||||
# make sure the classes.jar and javalib.jar are built before $(LOCAL_BUILT_MODULE)
|
# make sure the classes.jar and javalib.jar are built before $(LOCAL_BUILT_MODULE)
|
||||||
$(built_module) : $(common_javalib_jar)
|
$(built_module) : $(common_javalib_jar)
|
||||||
endif # TARGET JAVA_LIBRARIES
|
endif # TARGET JAVA_LIBRARIES
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Copyright 2015 The Android Open Source Project
|
||||||
|
#
|
||||||
|
# The rest of files in this directory comes from
|
||||||
|
# https://github.com/bazelbuild/bazel/tree/master/third_party/ijar
|
||||||
|
|
||||||
|
LOCAL_PATH:= $(call my-dir)
|
||||||
|
|
||||||
|
include $(CLEAR_VARS)
|
||||||
|
LOCAL_CPP_EXTENSION := cc
|
||||||
|
LOCAL_SRC_FILES := classfile.cc ijar.cc zip.cc
|
||||||
|
LOCAL_CFLAGS += -Wall
|
||||||
|
LOCAL_SHARED_LIBRARIES := libz-host
|
||||||
|
LOCAL_MODULE := ijar
|
||||||
|
include $(BUILD_HOST_EXECUTABLE)
|
||||||
|
|
@ -0,0 +1,203 @@
|
||||||
|
|
||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||||
|
replaced with your own identifying information. (Don't include
|
||||||
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
|
comment syntax for the file format. We also recommend that a
|
||||||
|
file or class name and description of purpose be included on the
|
||||||
|
same "printed page" as the copyright notice for easier
|
||||||
|
identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner]
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
|
||||||
|
|
@ -0,0 +1,120 @@
|
||||||
|
|
||||||
|
ijar: A tool for generating interface .jars from normal .jars
|
||||||
|
=============================================================
|
||||||
|
|
||||||
|
Alan Donovan, 26 May 2007.
|
||||||
|
|
||||||
|
Rationale:
|
||||||
|
|
||||||
|
In order to improve the speed of compilation of Java programs in
|
||||||
|
Bazel, the output of build steps is cached.
|
||||||
|
|
||||||
|
This works very nicely for C++ compilation: a compilation unit
|
||||||
|
includes a .cc source file and typically dozens of header files.
|
||||||
|
Header files change relatively infrequently, so the need for a
|
||||||
|
rebuild is usually driven by a change in the .cc file. Even after
|
||||||
|
syncing a slightly newer version of the tree and doing a rebuild,
|
||||||
|
many hits in the cache are still observed.
|
||||||
|
|
||||||
|
In Java, by contrast, a compilation unit involves a set of .java
|
||||||
|
source files, plus a set of .jar files containing already-compiled
|
||||||
|
JVM .class files. Class files serve a dual purpose: from the JVM's
|
||||||
|
perspective, they are containers of executable code, but from the
|
||||||
|
compiler's perspective, they are interface definitions. The problem
|
||||||
|
here is that .jar files are very much more sensitive to change than
|
||||||
|
C++ header files, so even a change that is insignificant to the
|
||||||
|
compiler (such as the addition of a print statement to a method in a
|
||||||
|
prerequisite class) will cause the jar to change, and any code that
|
||||||
|
depends on this jar's interface will be recompiled unnecessarily.
|
||||||
|
|
||||||
|
The purpose of ijar is to produce, from a .jar file, a much smaller,
|
||||||
|
simpler .jar file containing only the parts that are significant for
|
||||||
|
the purposes of compilation. In other words, an interface .jar
|
||||||
|
file. By changing ones compilation dependencies to be the interface
|
||||||
|
jar files, unnecessary recompilation is avoided when upstream
|
||||||
|
changes don't affect the interface.
|
||||||
|
|
||||||
|
Details:
|
||||||
|
|
||||||
|
ijar is a tool that reads a .jar file and emits a .jar file
|
||||||
|
containing only the parts that are relevant to Java compilation.
|
||||||
|
For example, it throws away:
|
||||||
|
|
||||||
|
- Files whose name does not end in ".class".
|
||||||
|
- All executable method code.
|
||||||
|
- All private methods and fields.
|
||||||
|
- All constants and attributes except the minimal set necessary to
|
||||||
|
describe the class interface.
|
||||||
|
- All debugging information
|
||||||
|
(LineNumberTable, SourceFile, LocalVariableTables attributes).
|
||||||
|
|
||||||
|
It also sets to zero the file modification times in the index of the
|
||||||
|
.jar file.
|
||||||
|
|
||||||
|
Implementation:
|
||||||
|
|
||||||
|
ijar is implemented in C++, and runs very quickly. For example
|
||||||
|
(when optimized) it takes only 530ms to process a 42MB
|
||||||
|
.jar file containing 5878 classe, resulting in an interface .jar
|
||||||
|
file of only 11.4MB in size. For more usual .jar sizes of a few
|
||||||
|
megabytes, a runtime of 50ms is typical.
|
||||||
|
|
||||||
|
The implementation strategy is to mmap both the input jar and the
|
||||||
|
newly-created _interface.jar, and to scan through the former and
|
||||||
|
emit the latter in a single pass. There are a couple of locations
|
||||||
|
where some kind of "backpatching" is required:
|
||||||
|
|
||||||
|
- in the .zip file format, for each file, the size field precedes
|
||||||
|
the data. We emit a zero but note its location, generate and emit
|
||||||
|
the stripped classfile, then poke the correct size into the
|
||||||
|
location.
|
||||||
|
|
||||||
|
- for JVM .class files, the header (including the constant table)
|
||||||
|
precedes the body, but cannot be emitted before it because it's
|
||||||
|
not until we emit the body that we know which constants are
|
||||||
|
referenced and which are garbage. So we emit the body into a
|
||||||
|
temporary buffer, then emit the header to the output jar, followed
|
||||||
|
by the contents of the temp buffer.
|
||||||
|
|
||||||
|
Also note that the zip file format has unnecessary duplication of
|
||||||
|
the index metadata: it has header+data for each file, then another
|
||||||
|
set of (similar) headers at the end. Rather than save the metadata
|
||||||
|
explicitly in some datastructure, we just record the addresses of
|
||||||
|
the already-emitted zip metadata entries in the output file, and
|
||||||
|
then read from there as necessary.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
|
||||||
|
This code has no dependency except on the STL and on zlib.
|
||||||
|
|
||||||
|
Almost all of the getX/putX/ReadX/WriteX functions in the code
|
||||||
|
advance their first argument pointer, which is passed by reference.
|
||||||
|
|
||||||
|
It's tempting to discard package-private classes and class members.
|
||||||
|
However, this would be incorrect because they are a necessary part
|
||||||
|
of the package interface, as a Java package is often compiled in
|
||||||
|
multiple stages. For example: in Bazel, both java tests and java
|
||||||
|
code inhabit the same Java package but are compiled separately.
|
||||||
|
|
||||||
|
Assumptions:
|
||||||
|
|
||||||
|
We assume that jar files are uncompressed v1.0 zip files (created
|
||||||
|
with 'jar c0f') with a zero general_purpose_bit_flag.
|
||||||
|
|
||||||
|
We assume that javap/javac don't need the correct CRC checksums in
|
||||||
|
the .jar file.
|
||||||
|
|
||||||
|
We assume that it's better simply to abort in the face of unknown
|
||||||
|
input than to risk leaving out something important from the output
|
||||||
|
(although in the case of annotations, it should be safe to ignore
|
||||||
|
ones we don't understand).
|
||||||
|
|
||||||
|
TODO:
|
||||||
|
Maybe: ensure a canonical sort order is used for every list (jar
|
||||||
|
entries, class members, attributes, etc.) This isn't essential
|
||||||
|
because we can assume the compiler is deterministic and the order in
|
||||||
|
the source files changes little. Also, it would require two passes. :(
|
||||||
|
|
||||||
|
Maybe: delete dynamically-allocated memory.
|
||||||
|
|
||||||
|
Add (a lot) more tests. Include a test of idempotency.
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,102 @@
|
||||||
|
// Copyright 2001,2007 Alan Donovan. All rights reserved.
|
||||||
|
//
|
||||||
|
// Author: Alan Donovan <adonovan@google.com>
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
//
|
||||||
|
// common.h -- common definitions.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef INCLUDED_DEVTOOLS_IJAR_COMMON_H
|
||||||
|
#define INCLUDED_DEVTOOLS_IJAR_COMMON_H
|
||||||
|
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
namespace devtools_ijar {
|
||||||
|
|
||||||
|
typedef unsigned long long u8;
|
||||||
|
typedef uint32_t u4;
|
||||||
|
typedef uint16_t u2;
|
||||||
|
typedef uint8_t u1;
|
||||||
|
|
||||||
|
// be = big endian, le = little endian
|
||||||
|
|
||||||
|
inline u1 get_u1(const u1 *&p) {
|
||||||
|
return *p++;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline u2 get_u2be(const u1 *&p) {
|
||||||
|
u4 x = (p[0] << 8) | p[1];
|
||||||
|
p += 2;
|
||||||
|
return x;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline u2 get_u2le(const u1 *&p) {
|
||||||
|
u4 x = (p[1] << 8) | p[0];
|
||||||
|
p += 2;
|
||||||
|
return x;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline u4 get_u4be(const u1 *&p) {
|
||||||
|
u4 x = (p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3];
|
||||||
|
p += 4;
|
||||||
|
return x;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline u4 get_u4le(const u1 *&p) {
|
||||||
|
u4 x = (p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0];
|
||||||
|
p += 4;
|
||||||
|
return x;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void put_u1(u1 *&p, u1 x) {
|
||||||
|
*p++ = x;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void put_u2be(u1 *&p, u2 x) {
|
||||||
|
*p++ = x >> 8;
|
||||||
|
*p++ = x & 0xff;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void put_u2le(u1 *&p, u2 x) {
|
||||||
|
*p++ = x & 0xff;
|
||||||
|
*p++ = x >> 8;;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void put_u4be(u1 *&p, u4 x) {
|
||||||
|
*p++ = x >> 24;
|
||||||
|
*p++ = (x >> 16) & 0xff;
|
||||||
|
*p++ = (x >> 8) & 0xff;
|
||||||
|
*p++ = x & 0xff;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void put_u4le(u1 *&p, u4 x) {
|
||||||
|
*p++ = x & 0xff;
|
||||||
|
*p++ = (x >> 8) & 0xff;
|
||||||
|
*p++ = (x >> 16) & 0xff;
|
||||||
|
*p++ = x >> 24;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy n bytes from src to p, and advance p.
|
||||||
|
inline void put_n(u1 *&p, const u1 *src, size_t n) {
|
||||||
|
memcpy(p, src, n);
|
||||||
|
p += n;
|
||||||
|
}
|
||||||
|
|
||||||
|
extern bool verbose;
|
||||||
|
|
||||||
|
} // namespace devtools_ijar
|
||||||
|
|
||||||
|
#endif // INCLUDED_DEVTOOLS_IJAR_COMMON_H
|
||||||
|
|
@ -0,0 +1,182 @@
|
||||||
|
// Copyright 2001,2007 Alan Donovan. All rights reserved.
|
||||||
|
//
|
||||||
|
// Author: Alan Donovan <adonovan@google.com>
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
//
|
||||||
|
// ijar.cpp -- .jar -> _interface.jar tool.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <limits.h>
|
||||||
|
#include <errno.h>
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
|
#include "zip.h"
|
||||||
|
|
||||||
|
namespace devtools_ijar {
|
||||||
|
|
||||||
|
bool verbose = false;
|
||||||
|
|
||||||
|
// Reads a JVM class from classdata_in (of the specified length), and
|
||||||
|
// writes out a simplified class to classdata_out, advancing the
|
||||||
|
// pointer.
|
||||||
|
void StripClass(u1 *&classdata_out, const u1 *classdata_in, size_t in_length);
|
||||||
|
|
||||||
|
const char* CLASS_EXTENSION = ".class";
|
||||||
|
const size_t CLASS_EXTENSION_LENGTH = strlen(CLASS_EXTENSION);
|
||||||
|
|
||||||
|
// ZipExtractorProcessor that select only .class file and use
|
||||||
|
// StripClass to generate an interface class, storing as a new file
|
||||||
|
// in the specified ZipBuilder.
|
||||||
|
class JarStripperProcessor : public ZipExtractorProcessor {
|
||||||
|
public:
|
||||||
|
JarStripperProcessor() {}
|
||||||
|
virtual ~JarStripperProcessor() {}
|
||||||
|
|
||||||
|
virtual void Process(const char* filename, const u4 attr,
|
||||||
|
const u1* data, const size_t size);
|
||||||
|
virtual bool Accept(const char* filename, const u4 attr);
|
||||||
|
|
||||||
|
private:
|
||||||
|
// Not owned by JarStripperProcessor, see SetZipBuilder().
|
||||||
|
ZipBuilder* builder;
|
||||||
|
|
||||||
|
public:
|
||||||
|
// Set the ZipBuilder to add the ijar class to the output zip file.
|
||||||
|
// This pointer should not be deleted while this class is still in use and
|
||||||
|
// it should be set before any call to the Process() method.
|
||||||
|
void SetZipBuilder(ZipBuilder* builder) {
|
||||||
|
this->builder = builder;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
bool JarStripperProcessor::Accept(const char* filename, const u4) {
|
||||||
|
ssize_t offset = strlen(filename) - CLASS_EXTENSION_LENGTH;
|
||||||
|
if (offset >= 0) {
|
||||||
|
return strcmp(filename + offset, CLASS_EXTENSION) == 0;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void JarStripperProcessor::Process(const char* filename, const u4,
|
||||||
|
const u1* data, const size_t size) {
|
||||||
|
if (verbose) {
|
||||||
|
fprintf(stderr, "INFO: StripClass: %s\n", filename);
|
||||||
|
}
|
||||||
|
u1 *q = builder->NewFile(filename, 0);
|
||||||
|
u1 *classdata_out = q;
|
||||||
|
StripClass(q, data, size); // actually process it
|
||||||
|
size_t out_length = q - classdata_out;
|
||||||
|
builder->FinishFile(out_length);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Opens "file_in" (a .jar file) for reading, and writes an interface
|
||||||
|
// .jar to "file_out".
|
||||||
|
void OpenFilesAndProcessJar(const char *file_out, const char *file_in) {
|
||||||
|
JarStripperProcessor processor;
|
||||||
|
std::unique_ptr<ZipExtractor> in(ZipExtractor::Create(file_in, &processor));
|
||||||
|
if (in.get() == NULL) {
|
||||||
|
fprintf(stderr, "Unable to open Zip file %s: %s\n", file_in,
|
||||||
|
strerror(errno));
|
||||||
|
abort();
|
||||||
|
}
|
||||||
|
u8 output_length = in->CalculateOutputLength();
|
||||||
|
std::unique_ptr<ZipBuilder> out(ZipBuilder::Create(file_out, output_length));
|
||||||
|
if (out.get() == NULL) {
|
||||||
|
fprintf(stderr, "Unable to open output file %s: %s\n", file_out,
|
||||||
|
strerror(errno));
|
||||||
|
abort();
|
||||||
|
}
|
||||||
|
processor.SetZipBuilder(out.get());
|
||||||
|
|
||||||
|
// Process all files in the zip
|
||||||
|
if (in->ProcessAll() < 0) {
|
||||||
|
fprintf(stderr, "%s\n", in->GetError());
|
||||||
|
abort();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add dummy file, since javac doesn't like truly empty jars.
|
||||||
|
if (out->GetNumberFiles() == 0) {
|
||||||
|
out->WriteEmptyFile("dummy");
|
||||||
|
}
|
||||||
|
// Finish writing the output file
|
||||||
|
if (out->Finish() < 0) {
|
||||||
|
fprintf(stderr, "%s\n", out->GetError());
|
||||||
|
abort();
|
||||||
|
}
|
||||||
|
// Get all file size
|
||||||
|
size_t in_length = in->GetSize();
|
||||||
|
size_t out_length = out->GetSize();
|
||||||
|
if (verbose) {
|
||||||
|
fprintf(stderr, "INFO: produced interface jar: %s -> %s (%d%%).\n",
|
||||||
|
file_in, file_out,
|
||||||
|
static_cast<int>(100.0 * out_length / in_length));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace devtools_ijar
|
||||||
|
|
||||||
|
//
|
||||||
|
// main method
|
||||||
|
//
|
||||||
|
static void usage() {
|
||||||
|
fprintf(stderr, "Usage: ijar [-v] x.jar [x_interface.jar>]\n");
|
||||||
|
fprintf(stderr, "Creates an interface jar from the specified jar file.\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char **argv) {
|
||||||
|
const char *filename_in = NULL;
|
||||||
|
const char *filename_out = NULL;
|
||||||
|
|
||||||
|
for (int ii = 1; ii < argc; ++ii) {
|
||||||
|
if (strcmp(argv[ii], "-v") == 0) {
|
||||||
|
devtools_ijar::verbose = true;
|
||||||
|
} else if (filename_in == NULL) {
|
||||||
|
filename_in = argv[ii];
|
||||||
|
} else if (filename_out == NULL) {
|
||||||
|
filename_out = argv[ii];
|
||||||
|
} else {
|
||||||
|
usage();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (filename_in == NULL) {
|
||||||
|
usage();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Guess output filename from input:
|
||||||
|
char filename_out_buf[PATH_MAX];
|
||||||
|
if (filename_out == NULL) {
|
||||||
|
size_t len = strlen(filename_in);
|
||||||
|
if (len > 4 && strncmp(filename_in + len - 4, ".jar", 4) == 0) {
|
||||||
|
strcpy(filename_out_buf, filename_in);
|
||||||
|
strcpy(filename_out_buf + len - 4, "-interface.jar");
|
||||||
|
filename_out = filename_out_buf;
|
||||||
|
} else {
|
||||||
|
fprintf(stderr, "Can't determine output filename since input filename "
|
||||||
|
"doesn't end with '.jar'.\n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (devtools_ijar::verbose) {
|
||||||
|
fprintf(stderr, "INFO: writing to '%s'.\n", filename_out);
|
||||||
|
}
|
||||||
|
|
||||||
|
devtools_ijar::OpenFilesAndProcessJar(filename_out, filename_in);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,173 @@
|
||||||
|
// Copyright 2015 Google Inc. All rights reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
//
|
||||||
|
// zip.h -- .zip (.jar) file reading/writing routines.
|
||||||
|
//
|
||||||
|
// This file specifies the interface to use the ZIP implementation of ijar.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef INCLUDED_THIRD_PARTY_IJAR_ZIP_H
|
||||||
|
#define INCLUDED_THIRD_PARTY_IJAR_ZIP_H
|
||||||
|
|
||||||
|
#include <sys/stat.h>
|
||||||
|
|
||||||
|
#include "common.h"
|
||||||
|
|
||||||
|
namespace devtools_ijar {
|
||||||
|
|
||||||
|
// Tells if this is a directory entry from the mode. This method
|
||||||
|
// is safer than zipattr_to_mode(attr) & S_IFDIR because the unix
|
||||||
|
// mode might not be set in DOS zip files.
|
||||||
|
inline bool zipattr_is_dir(u4 attr) { return (attr & 0x10) != 0; }
|
||||||
|
|
||||||
|
// Convert a Unix file mode to a ZIP file attribute
|
||||||
|
inline u4 mode_to_zipattr(mode_t m) {
|
||||||
|
return (((u4) m) << 16) + ((m & S_IFDIR) != 0 ? 0x10 : 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert a ZIP file attribute to a Unix file mode
|
||||||
|
inline mode_t zipattr_to_mode(u4 attr) {
|
||||||
|
return ((mode_t) ((attr >> 16) & 0xffff));
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
// Class interface for building ZIP files
|
||||||
|
//
|
||||||
|
class ZipBuilder {
|
||||||
|
public:
|
||||||
|
virtual ~ZipBuilder() {}
|
||||||
|
|
||||||
|
// Returns the text for the last error, or null on no last error.
|
||||||
|
virtual const char* GetError() = 0;
|
||||||
|
|
||||||
|
// Add a new file to the ZIP, the file will have path "filename"
|
||||||
|
// and external attributes "attr". This function returns a pointer
|
||||||
|
// to a memory buffer to write the data of the file into. This buffer
|
||||||
|
// is owned by ZipBuilder and should not be free'd by the caller. The
|
||||||
|
// file length is then specified when the files is finished written
|
||||||
|
// using the FinishFile(size_t) function.
|
||||||
|
// On failure, returns NULL and GetError() will return an non-empty message.
|
||||||
|
virtual u1* NewFile(const char* filename, const u4 attr) = 0;
|
||||||
|
|
||||||
|
// Finish writing a file and specify its length. After calling this method
|
||||||
|
// one should not reuse the pointer given by NewFile. The file can be
|
||||||
|
// compressed using the deflate algorithm by setting `compress` to true.
|
||||||
|
// By default, CRC32 are not computed as java tooling doesn't care, but
|
||||||
|
// computing it can be activated by setting `compute_crc` to true.
|
||||||
|
// On failure, returns -1 and GetError() will return an non-empty message.
|
||||||
|
virtual int FinishFile(size_t filelength,
|
||||||
|
bool compress = false,
|
||||||
|
bool compute_crc = false) = 0;
|
||||||
|
|
||||||
|
// Write an empty file, it is equivalent to:
|
||||||
|
// NewFile(filename, 0);
|
||||||
|
// FinishFile(0);
|
||||||
|
// On failure, returns -1 and GetError() will return an non-empty message.
|
||||||
|
virtual int WriteEmptyFile(const char* filename) = 0;
|
||||||
|
|
||||||
|
// Finish writing the ZIP file. This method can be called only once
|
||||||
|
// (subsequent calls will do nothing) and none of
|
||||||
|
// NewFile/FinishFile/WriteEmptyFile should be called after calling Finish. If
|
||||||
|
// this method was not called when the object is destroyed, it will be called.
|
||||||
|
// It is here as a convenience to get information on the final generated ZIP
|
||||||
|
// file.
|
||||||
|
// On failure, returns -1 and GetError() will return an non-empty message.
|
||||||
|
virtual int Finish() = 0;
|
||||||
|
|
||||||
|
// Get the current size of the ZIP file. This size will not be matching the
|
||||||
|
// final ZIP file until Finish() has been called because Finish() is actually
|
||||||
|
// writing the central directory of the ZIP File.
|
||||||
|
virtual size_t GetSize() = 0;
|
||||||
|
|
||||||
|
// Returns the current number of files stored in the ZIP.
|
||||||
|
virtual int GetNumberFiles() = 0;
|
||||||
|
|
||||||
|
// Create a new ZipBuilder writing the file zip_file and the size of the
|
||||||
|
// output will be at most estimated_size. Use ZipBuilder::EstimateSize() or
|
||||||
|
// ZipExtractor::CalculateOuputLength() to have an estimated_size depending on
|
||||||
|
// a list of file to store.
|
||||||
|
// On failure, returns NULL. Refer to errno for error code.
|
||||||
|
static ZipBuilder* Create(const char* zip_file, u8 estimated_size);
|
||||||
|
|
||||||
|
// Estimate the maximum size of the ZIP files containing files in the "files"
|
||||||
|
// null-terminated array.
|
||||||
|
// Returns 0 on error.
|
||||||
|
static u8 EstimateSize(char **files);
|
||||||
|
};
|
||||||
|
|
||||||
|
//
|
||||||
|
// An abstract class to process data from a ZipExtractor.
|
||||||
|
// Derive from this class if you wish to process data from a ZipExtractor.
|
||||||
|
//
|
||||||
|
class ZipExtractorProcessor {
|
||||||
|
public:
|
||||||
|
virtual ~ZipExtractorProcessor() {}
|
||||||
|
|
||||||
|
// Tells whether to skip or process the file "filename". "attr" is the
|
||||||
|
// external file attributes and can be converted to unix mode using the
|
||||||
|
// zipattr_to_mode() function. This method is suppoed to returns true
|
||||||
|
// if the file should be processed and false if it should be skipped.
|
||||||
|
virtual bool Accept(const char* filename, const u4 attr) = 0;
|
||||||
|
|
||||||
|
// Process a file accepted by Accept. The file "filename" has external
|
||||||
|
// attributes "attr" and length "size". The file content is accessible
|
||||||
|
// in the buffer pointed by "data".
|
||||||
|
virtual void Process(const char* filename, const u4 attr,
|
||||||
|
const u1* data, const size_t size) = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
//
|
||||||
|
// Class interface for reading ZIP files
|
||||||
|
//
|
||||||
|
class ZipExtractor {
|
||||||
|
public:
|
||||||
|
virtual ~ZipExtractor() {}
|
||||||
|
|
||||||
|
// Returns the text for the last error, or null on no last error.
|
||||||
|
virtual const char* GetError() = 0;
|
||||||
|
|
||||||
|
// Process the next files, returns false if the end of ZIP file has been
|
||||||
|
// reached. The processor provided by the Create method will be called
|
||||||
|
// if a file is encountered. If false is returned, check the return value
|
||||||
|
// of GetError() for potential errors.
|
||||||
|
virtual bool ProcessNext() = 0;
|
||||||
|
|
||||||
|
// Process the all files, returns -1 on error (GetError() will be populated
|
||||||
|
// on error).
|
||||||
|
virtual int ProcessAll();
|
||||||
|
|
||||||
|
// Reset the file pointer to the beginning.
|
||||||
|
virtual void Reset() = 0;
|
||||||
|
|
||||||
|
// Return the size of the ZIP file.
|
||||||
|
virtual size_t GetSize() = 0;
|
||||||
|
|
||||||
|
// Return the size of the resulting zip file by keeping only file
|
||||||
|
// accepted by the processor and storing them uncompressed. This
|
||||||
|
// method can be used to create a ZipBuilder for storing a subset
|
||||||
|
// of the input files.
|
||||||
|
// On error, 0 is returned and GetError() returns a non-empty message.
|
||||||
|
virtual u8 CalculateOutputLength() = 0;
|
||||||
|
|
||||||
|
// Create a ZipExtractor that extract the zip file "filename" and process
|
||||||
|
// it with "processor".
|
||||||
|
// On error, a null pointer is returned and the value of errno should be
|
||||||
|
// checked.
|
||||||
|
static ZipExtractor* Create(const char* filename,
|
||||||
|
ZipExtractorProcessor *processor);
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace devtools_ijar
|
||||||
|
|
||||||
|
#endif // INCLUDED_THIRD_PARTY_IJAR_ZIP_H
|
||||||
|
|
@ -0,0 +1,312 @@
|
||||||
|
// Copyright 2015 Google Inc. All rights reserved.
|
||||||
|
//
|
||||||
|
// Author: Alan Donovan <adonovan@google.com>
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
//
|
||||||
|
// Zip / Unzip file using ijar zip implementation.
|
||||||
|
//
|
||||||
|
// Note that this Zip implementation intentionally don't compute CRC-32
|
||||||
|
// because it is useless computation for jar because Java doesn't care.
|
||||||
|
// CRC-32 of all files in the zip file will be set to 0.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <limits.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <sys/mman.h>
|
||||||
|
#include <errno.h>
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
|
#include "zip.h"
|
||||||
|
|
||||||
|
namespace devtools_ijar {
|
||||||
|
|
||||||
|
#define SYSCALL(expr) do { \
|
||||||
|
if ((expr) < 0) { \
|
||||||
|
perror(#expr); \
|
||||||
|
abort(); \
|
||||||
|
} \
|
||||||
|
} while (0)
|
||||||
|
|
||||||
|
//
|
||||||
|
// A ZipExtractorProcessor that extract all files in the ZIP file.
|
||||||
|
//
|
||||||
|
class UnzipProcessor : public ZipExtractorProcessor {
|
||||||
|
public:
|
||||||
|
// Create a processor who will extract the files into output_root
|
||||||
|
// if "extract" is set to true and will print the list of files and
|
||||||
|
// their unix modes if "verbose" is set to true.
|
||||||
|
UnzipProcessor(const char *output_root, bool verbose, bool extract)
|
||||||
|
: output_root_(output_root), verbose_(verbose), extract_(extract) {}
|
||||||
|
virtual ~UnzipProcessor() {}
|
||||||
|
|
||||||
|
virtual void Process(const char* filename, const u4 attr,
|
||||||
|
const u1* data, const size_t size);
|
||||||
|
virtual bool Accept(const char* filename, const u4 attr) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
const char *output_root_;
|
||||||
|
const bool verbose_;
|
||||||
|
const bool extract_;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Concatene 2 path, path1 and path2, using / as a directory separator and
|
||||||
|
// puting the result in "out". "size" specify the size of the output buffer
|
||||||
|
void concat_path(char* out, const size_t size,
|
||||||
|
const char *path1, const char *path2) {
|
||||||
|
int len1 = strlen(path1);
|
||||||
|
size_t l = len1;
|
||||||
|
strncpy(out, path1, size - 1);
|
||||||
|
out[size-1] = 0;
|
||||||
|
if (l < size - 1 && path1[len1] != '/' && path2[0] != '/') {
|
||||||
|
out[l] = '/';
|
||||||
|
l++;
|
||||||
|
out[l] = 0;
|
||||||
|
}
|
||||||
|
if (l < size - 1) {
|
||||||
|
strncat(out, path2, size - 1 - l);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Do a recursive mkdir of all folders of path except the last path
|
||||||
|
// segment (if path ends with a / then the last path segment is empty).
|
||||||
|
// All folders are created using "mode" for creation mode.
|
||||||
|
void mkdirs(const char *path, mode_t mode) {
|
||||||
|
char path_[PATH_MAX];
|
||||||
|
struct stat statst;
|
||||||
|
strncpy(path_, path, PATH_MAX);
|
||||||
|
path_[PATH_MAX-1] = 0;
|
||||||
|
char *pointer = path_;
|
||||||
|
while ((pointer = strchr(pointer, '/')) != NULL) {
|
||||||
|
if (path_ != pointer) { // skip leading slash
|
||||||
|
*pointer = 0;
|
||||||
|
if (stat(path_, &statst) != 0) {
|
||||||
|
if (mkdir(path_, mode) < 0) {
|
||||||
|
fprintf(stderr, "Cannot create folder %s: %s\n",
|
||||||
|
path_, strerror(errno));
|
||||||
|
abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*pointer = '/';
|
||||||
|
}
|
||||||
|
pointer++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void UnzipProcessor::Process(const char* filename, const u4 attr,
|
||||||
|
const u1* data, const size_t size) {
|
||||||
|
mode_t mode = zipattr_to_mode(attr);
|
||||||
|
mode_t perm = mode & 0777;
|
||||||
|
bool isdir = (mode & S_IFDIR) != 0;
|
||||||
|
if (attr == 0) {
|
||||||
|
// Fallback when the external attribute is not set.
|
||||||
|
isdir = filename[strlen(filename)-1] == '/';
|
||||||
|
perm = 0777;
|
||||||
|
}
|
||||||
|
if (verbose_) {
|
||||||
|
printf("%c %o %s\n", isdir ? 'd' : 'f', perm, filename);
|
||||||
|
}
|
||||||
|
if (extract_) {
|
||||||
|
char path[PATH_MAX];
|
||||||
|
int fd;
|
||||||
|
concat_path(path, PATH_MAX, output_root_, filename);
|
||||||
|
mkdirs(path, perm);
|
||||||
|
if (!isdir) {
|
||||||
|
fd = open(path, O_CREAT | O_WRONLY, perm);
|
||||||
|
if (fd < 0) {
|
||||||
|
fprintf(stderr, "Cannot open file %s for writing: %s\n",
|
||||||
|
path, strerror(errno));
|
||||||
|
abort();
|
||||||
|
}
|
||||||
|
SYSCALL(write(fd, data, size));
|
||||||
|
SYSCALL(close(fd));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the basename of path and store it in output. output_size
|
||||||
|
// is the size of the output buffer.
|
||||||
|
void basename(const char *path, char *output, size_t output_size) {
|
||||||
|
const char *pointer = strrchr(path, '/');
|
||||||
|
if (pointer == NULL) {
|
||||||
|
pointer = path;
|
||||||
|
} else {
|
||||||
|
pointer++; // Skip the leading slash.
|
||||||
|
}
|
||||||
|
strncpy(output, pointer, output_size);
|
||||||
|
output[output_size-1] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Execute the extraction (or just listing if just v is provided)
|
||||||
|
int extract(char *zipfile, bool verbose, bool extract) {
|
||||||
|
char output_root[PATH_MAX];
|
||||||
|
getcwd(output_root, PATH_MAX);
|
||||||
|
|
||||||
|
UnzipProcessor processor(output_root, verbose, extract);
|
||||||
|
std::unique_ptr<ZipExtractor> extractor(ZipExtractor::Create(zipfile,
|
||||||
|
&processor));
|
||||||
|
if (extractor.get() == NULL) {
|
||||||
|
fprintf(stderr, "Unable to open zip file %s: %s.\n", zipfile,
|
||||||
|
strerror(errno));
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (extractor->ProcessAll() < 0) {
|
||||||
|
fprintf(stderr, "%s.\n", extractor->GetError());
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute the create operation
|
||||||
|
int create(char *zipfile, char **files, bool flatten, bool verbose,
|
||||||
|
bool compress) {
|
||||||
|
struct stat statst;
|
||||||
|
u8 size = ZipBuilder::EstimateSize(files);
|
||||||
|
if (size == 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
std::unique_ptr<ZipBuilder> builder(ZipBuilder::Create(zipfile, size));
|
||||||
|
if (builder.get() == NULL) {
|
||||||
|
fprintf(stderr, "Unable to create zip file %s: %s.\n",
|
||||||
|
zipfile, strerror(errno));
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
for (int i = 0; files[i] != NULL; i++) {
|
||||||
|
stat(files[i], &statst);
|
||||||
|
char path[PATH_MAX];
|
||||||
|
bool isdir = (statst.st_mode & S_IFDIR) != 0;
|
||||||
|
|
||||||
|
if (flatten && isdir) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute the path, flattening it if requested
|
||||||
|
if (flatten) {
|
||||||
|
basename(files[i], path, PATH_MAX);
|
||||||
|
} else {
|
||||||
|
strncpy(path, files[i], PATH_MAX);
|
||||||
|
path[PATH_MAX-1] = 0;
|
||||||
|
size_t len = strlen(path);
|
||||||
|
if (isdir && len < PATH_MAX - 1) {
|
||||||
|
// Add the trailing slash for folders
|
||||||
|
path[len] = '/';
|
||||||
|
path[len+1] = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (verbose) {
|
||||||
|
mode_t perm = statst.st_mode & 0777;
|
||||||
|
printf("%c %o %s\n", isdir ? 'd' : 'f', perm, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
u1 *buffer = builder->NewFile(path, mode_to_zipattr(statst.st_mode));
|
||||||
|
if (isdir || statst.st_size == 0) {
|
||||||
|
builder->FinishFile(0);
|
||||||
|
} else {
|
||||||
|
// mmap the input file and memcpy
|
||||||
|
int fd = open(files[i], O_RDONLY);
|
||||||
|
if (fd < 0) {
|
||||||
|
fprintf(stderr, "Can't open file %s for reading: %s.\n",
|
||||||
|
files[i], strerror(errno));
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
void *data = mmap(NULL, statst.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
|
||||||
|
if (data == MAP_FAILED) {
|
||||||
|
fprintf(stderr, "Can't mmap file %s for reading: %s.\n",
|
||||||
|
files[i], strerror(errno));
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
memcpy(buffer, data, statst.st_size);
|
||||||
|
munmap(data, statst.st_size);
|
||||||
|
builder->FinishFile(statst.st_size, compress, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (builder->Finish() < 0) {
|
||||||
|
fprintf(stderr, "%s\n", builder->GetError());
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace devtools_ijar
|
||||||
|
|
||||||
|
//
|
||||||
|
// main method
|
||||||
|
//
|
||||||
|
static void usage(char *progname) {
|
||||||
|
fprintf(stderr, "Usage: %s [vxc[fC]] x.zip [file1...filen]\n", progname);
|
||||||
|
fprintf(stderr, " v verbose - list all file in x.zip\n");
|
||||||
|
fprintf(stderr, " x extract - extract file in x.zip in current directory\n");
|
||||||
|
fprintf(stderr, " c create - add files to x.zip\n");
|
||||||
|
fprintf(stderr, " f flatten - flatten files to use with create operation\n");
|
||||||
|
fprintf(stderr,
|
||||||
|
" C compress - compress files when using the create operation\n");
|
||||||
|
fprintf(stderr, "x and c cannot be used in the same command-line.\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char **argv) {
|
||||||
|
bool extract = false;
|
||||||
|
bool verbose = false;
|
||||||
|
bool create = false;
|
||||||
|
bool compress = false;
|
||||||
|
bool flatten = false;
|
||||||
|
|
||||||
|
if (argc < 3) {
|
||||||
|
usage(argv[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; argv[1][i] != 0; i++) {
|
||||||
|
switch (argv[1][i]) {
|
||||||
|
case 'x':
|
||||||
|
extract = true;
|
||||||
|
break;
|
||||||
|
case 'v':
|
||||||
|
verbose = true;
|
||||||
|
break;
|
||||||
|
case 'c':
|
||||||
|
create = true;
|
||||||
|
break;
|
||||||
|
case 'f':
|
||||||
|
flatten = true;
|
||||||
|
break;
|
||||||
|
case 'C':
|
||||||
|
compress = true;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
usage(argv[0]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (create) {
|
||||||
|
if (extract) {
|
||||||
|
usage(argv[0]);
|
||||||
|
}
|
||||||
|
// Create a zip
|
||||||
|
return devtools_ijar::create(argv[2], argv + 3, flatten, verbose, compress);
|
||||||
|
} else {
|
||||||
|
if (flatten) {
|
||||||
|
usage(argv[0]);
|
||||||
|
}
|
||||||
|
// Extraction / list mode
|
||||||
|
return devtools_ijar::extract(argv[2], verbose, extract);
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue