From 3cd005d347b060f873b675e7a09e2aeb0e93eb25 Mon Sep 17 00:00:00 2001 From: Mohammad Samiul Islam Date: Thu, 26 Nov 2020 13:32:26 +0000 Subject: [PATCH] Enable soong build tool to handle APEX compression 1. Soong can now detect PRODUCT_COMPRESSED_APEX flag We don't want APEX to be compressed on all devices. Only those that have explicitely set PRODUCT_COMPRESSED_APEX flag. 2. Handle "compressible" field in soong build rule On devices that supports APEX compression, all APEX will be compressed by default. If any apex does not want to be compressed, they will need to state that by setting "compressible" field to false 3. Can use apex_compression_tool to compress APEX Note we compress the APEX after it has been signed. That way, when we decompress we will get a signed APEX. 4. Place the compressed APEX in system with .capex extension This makes it easy to identify. We still preserve the original extension so that when we decompress, we can just rename by cuttif off the .capex extension. Note: with this change, we can create a system image with compressed APEX, but we cannot boot with it since platform doesn't know how to handle .capex files. Platform support will be added on follow up CLs. Bug: 172911362 Test: OVERRIDE_PRODUCT_COMPRESSED_APEX=true m (apex_test.go) Test: observed $OUT/system/apex has .capex files Change-Id: I20ac4c4ceb521924c751a6017f979b2d808fdded --- android/config.go | 4 ++++ android/variable.go | 5 +++-- apex/androidmk.go | 6 +++++- apex/apex.go | 8 ++++++++ apex/apex_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ apex/builder.go | 38 +++++++++++++++++++++++++++++++++++--- 6 files changed, 96 insertions(+), 6 deletions(-) diff --git a/android/config.go b/android/config.go index 9882d5508..453e074c6 100644 --- a/android/config.go +++ b/android/config.go @@ -1272,6 +1272,10 @@ func (c *config) FlattenApex() bool { return Bool(c.productVariables.Flatten_apex) } +func (c *config) CompressedApex() bool { + return Bool(c.productVariables.CompressedApex) +} + func (c *config) EnforceSystemCertificate() bool { return Bool(c.productVariables.EnforceSystemCertificate) } diff --git a/android/variable.go b/android/variable.go index 0df5272c0..aed145c96 100644 --- a/android/variable.go +++ b/android/variable.go @@ -319,8 +319,9 @@ type productVariables struct { Ndk_abis *bool `json:",omitempty"` Exclude_draft_ndk_apis *bool `json:",omitempty"` - Flatten_apex *bool `json:",omitempty"` - Aml_abis *bool `json:",omitempty"` + Flatten_apex *bool `json:",omitempty"` + CompressedApex *bool `json:",omitempty"` + Aml_abis *bool `json:",omitempty"` DexpreoptGlobalConfig *string `json:",omitempty"` diff --git a/apex/androidmk.go b/apex/androidmk.go index da38c2ac8..6c76ad3f9 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -360,7 +360,11 @@ func (a *apexBundle) androidMkForType() android.AndroidMkData { fmt.Fprintln(w, "LOCAL_MODULE_CLASS := ETC") // do we need a new class? fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", a.outputFile.String()) fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", a.installDir.ToMakePath().String()) - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", name+apexType.suffix()) + stemSuffix := apexType.suffix() + if a.isCompressed { + stemSuffix = ".capex" + } + fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", name+stemSuffix) fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE :=", !a.installable()) // Because apex writes .mk with Custom(), we need to write manually some common properties diff --git a/apex/apex.go b/apex/apex.go index 7ab74541f..28ae4bd4a 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -120,6 +120,12 @@ type apexBundleProperties struct { // Default: true. Installable *bool + // Whether this APEX can be compressed or not. Setting this property to false means this + // APEX will never be compressed. When set to true, APEX will be compressed if other + // conditions, e.g, target device needs to support APEX compression, are also fulfilled. + // Default: true. + Compressible *bool + // For native libraries and binaries, use the vendor variant instead of the core (platform) // variant. Default is false. DO NOT use this for APEXes that are installed to the system or // system_ext partition. @@ -354,6 +360,8 @@ type apexBundle struct { prebuiltFileToDelete string + isCompressed bool + // Path of API coverage generate file coverageOutputPath android.ModuleOutPath } diff --git a/apex/apex_test.go b/apex/apex_test.go index 0b67ef577..7ffd2263a 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -346,6 +346,13 @@ func ensureListEmpty(t *testing.T, result []string) { } } +func ensureListNotEmpty(t *testing.T, result []string) { + t.Helper() + if len(result) == 0 { + t.Errorf("%q is expected to be not empty", result) + } +} + // Minimal test func TestBasicApex(t *testing.T) { ctx, config := testApex(t, ` @@ -6186,6 +6193,40 @@ func TestNonPreferredPrebuiltDependency(t *testing.T) { `) } +func TestCompressedApex(t *testing.T) { + ctx, config := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + compressible: true, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `, func(fs map[string][]byte, config android.Config) { + config.TestProductVariables.CompressedApex = proptools.BoolPtr(true) + }) + + compressRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("compressRule") + ensureContains(t, compressRule.Output.String(), "myapex.capex.unsigned") + + signApkRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Description("sign compressedApex") + ensureEquals(t, signApkRule.Input.String(), compressRule.Output.String()) + + // Make sure output of bundle is .capex + ab := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) + ensureContains(t, ab.outputFile.String(), "myapex.capex") + + // Verify android.mk rules + data := android.AndroidMkDataForTest(t, config, "", ab) + var builder strings.Builder + data.Custom(&builder, ab.BaseModuleName(), "TARGET_", "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE_STEM := myapex.capex\n") +} + func TestPreferredPrebuiltSharedLibDep(t *testing.T) { ctx, config := testApex(t, ` apex { diff --git a/apex/builder.go b/apex/builder.go index 66eaff1d7..9db8e5929 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -66,6 +66,7 @@ func init() { pctx.HostBinToolVariable("extract_apks", "extract_apks") pctx.HostBinToolVariable("make_f2fs", "make_f2fs") pctx.HostBinToolVariable("sload_f2fs", "sload_f2fs") + pctx.HostBinToolVariable("apex_compression_tool", "apex_compression_tool") pctx.SourcePathVariable("genNdkUsedbyApexPath", "build/soong/scripts/gen_ndk_usedby_apex.sh") } @@ -738,7 +739,7 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { //////////////////////////////////////////////////////////////////////////////////// // Step 4: Sign the APEX using signapk - a.outputFile = android.PathForModuleOut(ctx, a.Name()+suffix) + signedOutputFile := android.PathForModuleOut(ctx, a.Name()+suffix) pem, key := a.getCertificateAndPrivateKey(ctx) rule := java.Signapk @@ -750,16 +751,47 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_SIGNAPK") { rule = java.SignapkRE args["implicits"] = strings.Join(implicits.Strings(), ",") - args["outCommaList"] = a.outputFile.String() + args["outCommaList"] = signedOutputFile.String() } ctx.Build(pctx, android.BuildParams{ Rule: rule, Description: "signapk", - Output: a.outputFile, + Output: signedOutputFile, Input: unsignedOutputFile, Implicits: implicits, Args: args, }) + a.outputFile = signedOutputFile + + // Process APEX compression if enabled + compressionEnabled := ctx.Config().CompressedApex() && proptools.BoolDefault(a.properties.Compressible, true) + if compressionEnabled && apexType == imageApex { + a.isCompressed = true + unsignedCompressedOutputFile := android.PathForModuleOut(ctx, a.Name()+".capex.unsigned") + + compressRule := android.NewRuleBuilder(pctx, ctx) + compressRule.Command(). + Text("rm"). + FlagWithOutput("-f ", unsignedCompressedOutputFile) + compressRule.Command(). + BuiltTool("apex_compression_tool"). + Flag("compress"). + FlagWithArg("--apex_compression_tool ", outHostBinDir+":"+prebuiltSdkToolsBinDir). + FlagWithInput("--input ", signedOutputFile). + FlagWithOutput("--output ", unsignedCompressedOutputFile) + compressRule.Build("compressRule", "Generate unsigned compressed APEX file") + + signedCompressedOutputFile := android.PathForModuleOut(ctx, a.Name()+".capex") + ctx.Build(pctx, android.BuildParams{ + Rule: rule, + Description: "sign compressedApex", + Output: signedCompressedOutputFile, + Input: unsignedCompressedOutputFile, + Implicits: implicits, + Args: args, + }) + a.outputFile = signedCompressedOutputFile + } // Install to $OUT/soong/{target,host}/.../apex if a.installable() {