Rework how linux_bionic is built with LLD

In order to simplify the wrapper function, and stop using a linker
script, generate a set of flags to pass to LLD. Then run
host_bionic_inject on the linked binary in order to verify the
embedding, and give the wrapper function the address of the original
entry point (_start).

Bug: 31559095
Test: build host bionic with prebuilts/build-tools/build-prebuilts.sh
Change-Id: I53e326050e0f9caa562c6cf6f76c4d0337bb6faf

Android.bp

@@ -482,7 +482,7 @@ cc_genrule {
 }
 
 cc_genrule {
-    name: "host_bionic_linker_script",
+    name: "host_bionic_linker_flags",
     host_supported: true,
     device_supported: false,
     target: {
@@ -497,7 +497,7 @@ cc_genrule {
         },
     },
     tools: ["extract_linker"],
-    cmd: "$(location) -T $(out) $(in)",
+    cmd: "$(location) -f $(out) $(in)",
     srcs: [":linker"],
-    out: ["linker.script"],
+    out: ["linker.flags"],
 }

cc/binary.go

@@ -15,6 +15,8 @@
 package cc
 
 import (
+	"github.com/google/blueprint"
+
 	"android/soong/android"
 )
 
@@ -154,7 +156,8 @@ func (binary *binaryDecorator) linkerDeps(ctx DepsContext, deps Deps) Deps {
 		}
 
 		if ctx.Os() == android.LinuxBionic && !binary.static() {
-			deps.LinkerScript = "host_bionic_linker_script"
+			deps.DynamicLinker = "linker"
+			deps.LinkerFlagsFile = "host_bionic_linker_flags"
 		}
 	}
 
@@ -244,14 +247,23 @@ func (binary *binaryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags
 					switch ctx.Os() {
 					case android.Android:
 						flags.DynamicLinker = "/system/bin/linker"
+						if flags.Toolchain.Is64Bit() {
+							flags.DynamicLinker += "64"
+						}
 					case android.LinuxBionic:
 						flags.DynamicLinker = ""
 					default:
 						ctx.ModuleErrorf("unknown dynamic linker")
 					}
-					if flags.Toolchain.Is64Bit() {
+				}
-						flags.DynamicLinker += "64"
+
-					}
+				if ctx.Os() == android.LinuxBionic {
+					// Use the dlwrap entry point, but keep _start around so
+					// that it can be used by host_bionic_inject
+					flags.LdFlags = append(flags.LdFlags,
+						"-Wl,--entry=__dlwrap__start",
+						"-Wl,--undefined=_start",
+					)
 				}
 			}
 
@@ -262,7 +274,6 @@ func (binary *binaryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags
 				"-Wl,--gc-sections",
 				"-Wl,-z,nocopyreloc",
 			)
-
 		}
 	} else {
 		if binary.static() {
@@ -288,13 +299,15 @@ func (binary *binaryDecorator) link(ctx ModuleContext,
 	sharedLibs := deps.SharedLibs
 	sharedLibs = append(sharedLibs, deps.LateSharedLibs...)
 
-	if deps.LinkerScript.Valid() {
+	if deps.LinkerFlagsFile.Valid() {
-		flags.LdFlags = append(flags.LdFlags, "-Wl,-T,"+deps.LinkerScript.String())
+		flags.LdFlags = append(flags.LdFlags, "$$(cat "+deps.LinkerFlagsFile.String()+")")
-		linkerDeps = append(linkerDeps, deps.LinkerScript.Path())
+		linkerDeps = append(linkerDeps, deps.LinkerFlagsFile.Path())
 	}
 
 	if flags.DynamicLinker != "" {
-		flags.LdFlags = append(flags.LdFlags, " -Wl,-dynamic-linker,"+flags.DynamicLinker)
+		flags.LdFlags = append(flags.LdFlags, "-Wl,-dynamic-linker,"+flags.DynamicLinker)
+	} else if ctx.toolchain().Bionic() && !binary.static() {
+		flags.LdFlags = append(flags.LdFlags, "-Wl,--no-dynamic-linker")
 	}
 
 	builderFlags := flagsToBuilderFlags(flags)
@@ -323,6 +336,17 @@ func (binary *binaryDecorator) link(ctx ModuleContext,
 		binary.injectVersionSymbol(ctx, outputFile, versionedOutputFile)
 	}
 
+	if ctx.Os() == android.LinuxBionic && !binary.static() {
+		injectedOutputFile := outputFile
+		outputFile = android.PathForModuleOut(ctx, "prelinker", fileName)
+
+		if !deps.DynamicLinker.Valid() {
+			panic("Non-static host bionic modules must have a dynamic linker")
+		}
+
+		binary.injectHostBionicLinkerSymbols(ctx, outputFile, deps.DynamicLinker.Path(), injectedOutputFile)
+	}
+
 	linkerDeps = append(linkerDeps, deps.SharedLibsDeps...)
 	linkerDeps = append(linkerDeps, deps.LateSharedLibsDeps...)
 	linkerDeps = append(linkerDeps, objs.tidyFiles...)
@@ -367,3 +391,26 @@ func (binary *binaryDecorator) install(ctx ModuleContext, file android.Path) {
 func (binary *binaryDecorator) hostToolPath() android.OptionalPath {
 	return binary.toolPath
 }
+
+func init() {
+	pctx.HostBinToolVariable("hostBionicSymbolsInjectCmd", "host_bionic_inject")
+}
+
+var injectHostBionicSymbols = pctx.AndroidStaticRule("injectHostBionicSymbols",
+	blueprint.RuleParams{
+		Command:     "$hostBionicSymbolsInjectCmd -i $in -l $linker -o $out",
+		CommandDeps: []string{"$hostBionicSymbolsInjectCmd"},
+	}, "linker")
+
+func (binary *binaryDecorator) injectHostBionicLinkerSymbols(ctx ModuleContext, in, linker android.Path, out android.WritablePath) {
+	ctx.Build(pctx, android.BuildParams{
+		Rule:        injectHostBionicSymbols,
+		Description: "inject host bionic symbols",
+		Input:       in,
+		Implicit:    linker,
+		Output:      out,
+		Args: map[string]string{
+			"linker": linker.String(),
+		},
+	})
+}

cc/cc.go

@@ -83,7 +83,10 @@ type Deps struct {
 	ReexportGeneratedHeaders []string
 
 	CrtBegin, CrtEnd string
-	LinkerScript     string
+
+	// Used for host bionic
+	LinkerFlagsFile string
+	DynamicLinker   string
 }
 
 type PathDeps struct {
@@ -108,7 +111,12 @@ type PathDeps struct {
 
 	// Paths to crt*.o files
 	CrtBegin, CrtEnd android.OptionalPath
-	LinkerScript     android.OptionalPath
+
+	// Path to the file container flags to use with the linker
+	LinkerFlagsFile android.OptionalPath
+
+	// Path to the dynamic linker binary
+	DynamicLinker android.OptionalPath
 }
 
 type Flags struct {
@@ -306,7 +314,8 @@ var (
 	objDepTag             = dependencyTag{name: "obj"}
 	crtBeginDepTag        = dependencyTag{name: "crtbegin"}
 	crtEndDepTag          = dependencyTag{name: "crtend"}
-	linkerScriptDepTag    = dependencyTag{name: "linker script"}
+	linkerFlagsDepTag     = dependencyTag{name: "linker flags file"}
+	dynamicLinkerDepTag   = dependencyTag{name: "dynamic linker"}
 	reuseObjTag           = dependencyTag{name: "reuse objects"}
 	ndkStubDepTag         = dependencyTag{name: "ndk stub", library: true}
 	ndkLateStubDepTag     = dependencyTag{name: "ndk late stub", library: true}
@@ -1062,8 +1071,11 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) {
 	if deps.CrtEnd != "" {
 		actx.AddVariationDependencies(nil, crtEndDepTag, deps.CrtEnd)
 	}
-	if deps.LinkerScript != "" {
+	if deps.LinkerFlagsFile != "" {
-		actx.AddDependency(c, linkerScriptDepTag, deps.LinkerScript)
+		actx.AddDependency(c, linkerFlagsDepTag, deps.LinkerFlagsFile)
+	}
+	if deps.DynamicLinker != "" {
+		actx.AddDependency(c, dynamicLinkerDepTag, deps.DynamicLinker)
 	}
 
 	version := ctx.sdkVersion()
@@ -1257,13 +1269,13 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps {
 				} else {
 					ctx.ModuleErrorf("module %q is not a genrule", depName)
 				}
-			case linkerScriptDepTag:
+			case linkerFlagsDepTag:
 				if genRule, ok := dep.(genrule.SourceFileGenerator); ok {
 					files := genRule.GeneratedSourceFiles()
 					if len(files) == 1 {
-						depPaths.LinkerScript = android.OptionalPathForPath(files[0])
+						depPaths.LinkerFlagsFile = android.OptionalPathForPath(files[0])
 					} else if len(files) > 1 {
-						ctx.ModuleErrorf("module %q can only generate a single file if used for a linker script", depName)
+						ctx.ModuleErrorf("module %q can only generate a single file if used for a linker flag file", depName)
 					}
 				} else {
 					ctx.ModuleErrorf("module %q is not a genrule", depName)
@@ -1358,6 +1370,8 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps {
 			depPaths.CrtBegin = linkFile
 		case crtEndDepTag:
 			depPaths.CrtEnd = linkFile
+		case dynamicLinkerDepTag:
+			depPaths.DynamicLinker = linkFile
 		}
 
 		switch depTag {

cmd/extract_linker/main.go

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 // This tool extracts ELF LOAD segments from our linker binary, and produces an
-// assembly file and linker script which will embed those segments as sections
+// assembly file and linker flags which will embed those segments as sections
 // in another binary.
 package main
 
@@ -26,38 +26,15 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"text/template"
+	"strings"
 )
 
-var linkerScriptTemplate = template.Must(template.New("linker_script").Parse(`
-ENTRY(__dlwrap__start)
-SECTIONS {
-	__dlwrap_original_start = _start;
-	/DISCARD/ : { *(.interp) }
-
-{{range .}}
-	. = {{ printf "0x%x" .Vaddr }};
-	{{.Name}} : { KEEP(*({{.Name}})) }
-{{end}}
-
-	.text : { *(.text .text.*) }
-	.rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) }
-	.data : { *(.data .data.* .gnu.linkonce.d.*) }
-	.bss : { *(.dynbss) *(.bss .bss.* .gnu.linkonce.b.*) *(COMMON) }
-}
-`))
-
-type LinkerSection struct {
-	Name  string
-	Vaddr uint64
-}
-
 func main() {
 	var asmPath string
-	var scriptPath string
+	var flagsPath string
 
 	flag.StringVar(&asmPath, "s", "", "Path to save the assembly file")
-	flag.StringVar(&scriptPath, "T", "", "Path to save the linker script")
+	flag.StringVar(&flagsPath, "f", "", "Path to save the linker flags")
 	flag.Parse()
 
 	f, err := os.Open(flag.Arg(0))
@@ -72,19 +49,21 @@ func main() {
 	}
 
 	asm := &bytes.Buffer{}
-
-	fmt.Fprintln(asm, ".globl __dlwrap_linker_entry")
-	fmt.Fprintf(asm, ".set __dlwrap_linker_entry, 0x%x\n\n", ef.Entry)
-
 	baseLoadAddr := uint64(0x1000)
-	sections := []LinkerSection{}
 	load := 0
+	linkFlags := []string{}
+
+	fmt.Fprintln(asm, ".globl __dlwrap_linker_offset")
+	fmt.Fprintf(asm, ".set __dlwrap_linker_offset, 0x%x\n", baseLoadAddr)
+
 	for _, prog := range ef.Progs {
 		if prog.Type != elf.PT_LOAD {
 			continue
 		}
 
 		sectionName := fmt.Sprintf(".linker.sect%d", load)
+		symName := fmt.Sprintf("__dlwrap_linker_sect%d", load)
+
 		flags := ""
 		if prog.Flags&elf.PF_W != 0 {
 			flags += "w"
@@ -94,10 +73,12 @@ func main() {
 		}
 		fmt.Fprintf(asm, ".section %s, \"a%s\"\n", sectionName, flags)
 
-		if load == 0 {
+		fmt.Fprintf(asm, ".globl %s\n%s:\n\n", symName, symName)
-			fmt.Fprintln(asm, ".globl __dlwrap_linker_code_start")
+
-			fmt.Fprintln(asm, "__dlwrap_linker_code_start:")
+		linkFlags = append(linkFlags,
-		}
+			fmt.Sprintf("-Wl,--undefined=%s", symName),
+			fmt.Sprintf("-Wl,--section-start=%s=0x%x",
+				sectionName, baseLoadAddr+prog.Vaddr))
 
 		buffer, _ := ioutil.ReadAll(prog.Open())
 		bytesToAsm(asm, buffer)
@@ -113,11 +94,6 @@ func main() {
 		}
 		fmt.Fprintln(asm)
 
-		sections = append(sections, LinkerSection{
-			Name:  sectionName,
-			Vaddr: baseLoadAddr + prog.Vaddr,
-		})
-
 		load += 1
 	}
 
@@ -127,13 +103,10 @@ func main() {
 		}
 	}
 
-	if scriptPath != "" {
+	if flagsPath != "" {
-		buf := &bytes.Buffer{}
+		flags := strings.Join(linkFlags, " ")
-		if err := linkerScriptTemplate.Execute(buf, sections); err != nil {
+		if err := ioutil.WriteFile(flagsPath, []byte(flags), 0777); err != nil {
-			log.Fatalf("Failed to create linker script: %v", err)
+			log.Fatalf("Unable to write %q: %v", flagsPath, err)
-		}
-		if err := ioutil.WriteFile(scriptPath, buf.Bytes(), 0777); err != nil {
-			log.Fatalf("Unable to write %q: %v", scriptPath, err)
 		}
 	}
 }

cmd/host_bionic_inject/Android.bp

@@ -0,0 +1,19 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+blueprint_go_binary {
+    name: "host_bionic_inject",
+    deps: ["soong-symbol_inject"],
+    srcs: ["host_bionic_inject.go"],
+}

cmd/host_bionic_inject/host_bionic_inject.go

@@ -0,0 +1,174 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Verifies a host bionic executable with an embedded linker, then injects
+// the address of the _start function for the linker_wrapper to use.
+package main
+
+import (
+	"debug/elf"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"android/soong/symbol_inject"
+)
+
+func main() {
+	var inputFile, linkerFile, outputFile string
+
+	flag.StringVar(&inputFile, "i", "", "Input file")
+	flag.StringVar(&linkerFile, "l", "", "Linker file")
+	flag.StringVar(&outputFile, "o", "", "Output file")
+	flag.Parse()
+
+	if inputFile == "" || linkerFile == "" || outputFile == "" || flag.NArg() != 0 {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	r, err := os.Open(inputFile)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(2)
+	}
+	defer r.Close()
+
+	file, err := symbol_inject.OpenFile(r)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(3)
+	}
+
+	linker, err := elf.Open(linkerFile)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(4)
+	}
+
+	start_addr, err := parseElf(r, linker)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(5)
+	}
+
+	w, err := os.OpenFile(outputFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0777)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(6)
+	}
+	defer w.Close()
+
+	err = symbol_inject.InjectUint64Symbol(file, w, "__dlwrap_original_start", start_addr)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(7)
+	}
+}
+
+// Check the ELF file, and return the address to the _start function
+func parseElf(r io.ReaderAt, linker *elf.File) (uint64, error) {
+	file, err := elf.NewFile(r)
+	if err != nil {
+		return 0, err
+	}
+
+	symbols, err := file.Symbols()
+	if err != nil {
+		return 0, err
+	}
+
+	for _, prog := range file.Progs {
+		if prog.Type == elf.PT_INTERP {
+			return 0, fmt.Errorf("File should not have a PT_INTERP header")
+		}
+	}
+
+	if dlwrap_start, err := findSymbol(symbols, "__dlwrap__start"); err != nil {
+		return 0, err
+	} else if dlwrap_start.Value != file.Entry {
+		return 0, fmt.Errorf("Expected file entry(0x%x) to point to __dlwrap_start(0x%x)",
+			file.Entry, dlwrap_start.Value)
+	}
+
+	err = checkLinker(file, linker, symbols)
+	if err != nil {
+		return 0, err
+	}
+
+	start, err := findSymbol(symbols, "_start")
+	if err != nil {
+		return 0, fmt.Errorf("Failed to find _start symbol")
+	}
+	return start.Value, nil
+}
+
+func findSymbol(symbols []elf.Symbol, name string) (elf.Symbol, error) {
+	for _, sym := range symbols {
+		if sym.Name == name {
+			return sym, nil
+		}
+	}
+	return elf.Symbol{}, fmt.Errorf("Failed to find symbol %q", name)
+}
+
+// Check that all of the PT_LOAD segments have been embedded properly
+func checkLinker(file, linker *elf.File, fileSyms []elf.Symbol) error {
+	dlwrap_linker_offset, err := findSymbol(fileSyms, "__dlwrap_linker_offset")
+	if err != nil {
+		return err
+	}
+
+	for i, lprog := range linker.Progs {
+		if lprog.Type != elf.PT_LOAD {
+			continue
+		}
+
+		found := false
+		for j, prog := range file.Progs {
+			if prog.Type != elf.PT_LOAD {
+				continue
+			}
+
+			if lprog.Vaddr+dlwrap_linker_offset.Value != prog.Vaddr {
+				continue
+			}
+			found = true
+
+			if lprog.Memsz != prog.Memsz {
+				return fmt.Errorf("Linker prog %d (0x%x) memsz (0x%x) does not match (0x%x)",
+					i, lprog.Vaddr, lprog.Memsz, prog.Memsz)
+			}
+
+			// The linker shouldn't be using BSS, since only one
+			// BSS section is supported per ELF file.
+			if prog.Memsz != prog.Filesz {
+				return fmt.Errorf("Embedded prog %d (0x%x) memsz (0x%x) does not match filesz (0x%x)",
+					j, prog.Vaddr, prog.Memsz, prog.Filesz)
+			}
+
+			if lprog.Flags != prog.Flags {
+				return fmt.Errorf("Linker prog %d (0x%x) flags (%s) do not match (%s)",
+					i, lprog.Vaddr, lprog.Flags, prog.Flags)
+			}
+		}
+		if !found {
+			return fmt.Errorf("Linker prog %d (0x%x) not found at offset 0x%x",
+				i, lprog.Vaddr, dlwrap_linker_offset.Value)
+		}
+	}
+
+	return nil
+}