Merge changes I6e4f957c,Id41a2d81

* changes:
  Use makeparallel to propagate -j flag to ninja or kati
  Implement makeparallel

core/ninja.mk

@@ -1,4 +1,5 @@
 KATI ?= $(HOST_OUT_EXECUTABLES)/ckati
+MAKEPARALLEL ?= $(HOST_OUT_EXECUTABLES)/makeparallel
 
 KATI_OUTPUT_PATTERNS := $(PRODUCT_OUT)/build%.ninja $(PRODUCT_OUT)/ninja%.sh
 NINJA_GOALS := fastincremental generateonly droid showcommands
@@ -85,14 +86,20 @@ PHONY: showcommands
 showcommands: droid
 endif
 
+ifdef KATI_REMOTE_NUM_JOBS_FLAG
+KATI_MAKEPARALLEL := $(MAKEPARALLEL)
+else
+NINJA_MAKEPARALLEL := $(MAKEPARALLEL)
+endif
+
 ifeq (,$(filter generateonly,$(ORIGINAL_MAKECMDGOALS)))
 fastincremental droid $(ANDROID_TARGETS): ninja.intermediate
 	@#empty
 
 .INTERMEDIATE: ninja.intermediate
-ninja.intermediate: $(KATI_OUTPUTS)
+ninja.intermediate: $(KATI_OUTPUTS) $(MAKEPARALLEL)
 	@echo Starting build with ninja
-	$(hide) PATH=prebuilts/ninja/$(HOST_PREBUILT_TAG)/:$$PATH NINJA_STATUS="$(NINJA_STATUS)" $(KATI_NINJA_SH) -C $(TOP) $(NINJA_ARGS) $(ANDROID_TARGETS)
+	+$(hide) PATH=prebuilts/ninja/$(HOST_PREBUILT_TAG)/:$$PATH NINJA_STATUS="$(NINJA_STATUS)" $(NINJA_MAKEPARALLEL) $(KATI_NINJA_SH) -C $(TOP) $(NINJA_ARGS) $(ANDROID_TARGETS)
 else
 generateonly droid $(ANDROID_TARGETS): $(KATI_OUTPUTS)
 	@#empty
@@ -105,10 +112,10 @@ endif
 $(KATI_OUTPUTS): kati.intermediate $(KATI_FORCE)
 
 .INTERMEDIATE: kati.intermediate
-kati.intermediate: $(KATI)
+kati.intermediate: $(KATI) $(MAKEPARALLEL)
 	@echo Running kati to generate build$(KATI_NINJA_SUFFIX).ninja...
 	@#TODO: use separate ninja file for mm or single target build
-	$(hide) $(KATI) --ninja --ninja_dir=$(PRODUCT_OUT) --ninja_suffix=$(KATI_NINJA_SUFFIX) --regen --ignore_dirty=$(OUT_DIR)/% --ignore_optional_include=$(OUT_DIR)/%.P --detect_android_echo --use_find_emulator $(KATI_REMOTE_NUM_JOBS_FLAG) -f build/core/main.mk $(or $(KATI_TARGETS),--gen_all_phony_targets) USE_NINJA=false
+	+$(hide) $(KATI_MAKEPARALLEL) $(KATI) --ninja --ninja_dir=$(PRODUCT_OUT) --ninja_suffix=$(KATI_NINJA_SUFFIX) --regen --ignore_dirty=$(OUT_DIR)/% --ignore_optional_include=$(OUT_DIR)/%.P --detect_android_echo --use_find_emulator $(KATI_REMOTE_NUM_JOBS_FLAG) -f build/core/main.mk $(or $(KATI_TARGETS),--gen_all_phony_targets) USE_NINJA=false
 
 KATI_CXX := $(CLANG_CXX) $(CLANG_HOST_GLOBAL_CPPFLAGS)
 KATI_LD := $(CLANG_CXX) $(CLANG_HOST_GLOBAL_LDFLAGS)
@@ -121,5 +128,16 @@ KATI_INTERMEDIATES_PATH := $(HOST_OUT_INTERMEDIATES)/EXECUTABLES/ckati_intermedi
 KATI_BIN_PATH := $(HOST_OUT_EXECUTABLES)
 include build/kati/Makefile.ckati
 
+MAKEPARALLEL_CXX := $(CLANG_CXX) $(CLANG_HOST_GLOBAL_CPPFLAGS)
+MAKEPARALLEL_LD := $(CLANG_CXX) $(CLANG_HOST_GLOBAL_LDFLAGS)
+# Build static makeparallel. Unfortunately Mac OS X doesn't officially support static exectuables.
+ifeq ($(BUILD_OS),linux)
+MAKEPARALLEL_LD += -static
+endif
+
+MAKEPARALLEL_INTERMEDIATES_PATH := $(HOST_OUT_INTERMEDIATES)/EXECUTABLES/makeparallel_intermediates
+MAKEPARALLEL_BIN_PATH := $(HOST_OUT_EXECUTABLES)
+include build/tools/makeparallel/Makefile
+
 .PHONY: FORCE
 FORCE:

tools/makeparallel/.gitignore

@@ -0,0 +1,4 @@
+makeparallel
+*.o
+*.d
+test.out

tools/makeparallel/Makefile

@@ -0,0 +1,64 @@
+# Copyright 2015 Google Inc. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Find source file location from path to this Makefile
+MAKEPARALLEL_SRC_PATH := $(patsubst %/,%,$(dir $(lastword $(MAKEFILE_LIST))))
+ifndef MAKEPARALLEL_SRC_PATH
+  MAKEPARALLEL_SRC_PATH := .
+endif
+
+# Set defaults if they weren't set by the including Makefile
+MAKEPARALLEL_CXX ?= $(CXX)
+MAKEPARALLEL_LD ?= $(CXX)
+MAKEPARALLEL_INTERMEDIATES_PATH ?= .
+MAKEPARALLEL_BIN_PATH ?= .
+
+MAKEPARALLEL_CXX_SRCS := \
+	makeparallel.cpp
+
+MAKEPARALLEL_CXXFLAGS := -Wall -Werror -MMD -MP
+
+MAKEPARALLEL_CXX_SRCS := $(addprefix $(MAKEPARALLEL_SRC_PATH)/,\
+	$(MAKEPARALLEL_CXX_SRCS))
+
+MAKEPARALLEL_CXX_OBJS := $(patsubst $(MAKEPARALLEL_SRC_PATH)/%.cpp,$(MAKEPARALLEL_INTERMEDIATES_PATH)/%.o,$(MAKEPARALLEL_CXX_SRCS))
+
+MAKEPARALLEL := $(MAKEPARALLEL_BIN_PATH)/makeparallel
+
+ifeq ($(shell uname),Linux)
+MAKEPARALLEL_LIBS := -lrt -lpthread
+endif
+
+# Rule to build makeparallel into MAKEPARALLEL_BIN_PATH
+$(MAKEPARALLEL): $(MAKEPARALLEL_CXX_OBJS)
+	@mkdir -p $(dir $@)
+	$(MAKEPARALLEL_LD) -std=c++11 $(MAKEPARALLEL_CXXFLAGS) -o $@ $^ $(MAKEPARALLEL_LIBS)
+
+# Rule to build source files into object files in MAKEPARALLEL_INTERMEDIATES_PATH
+$(MAKEPARALLEL_CXX_OBJS): $(MAKEPARALLEL_INTERMEDIATES_PATH)/%.o: $(MAKEPARALLEL_SRC_PATH)/%.cpp
+	@mkdir -p $(dir $@)
+	$(MAKEPARALLEL_CXX) -c -std=c++11 $(MAKEPARALLEL_CXXFLAGS) -o $@ $<
+
+makeparallel_clean:
+	rm -rf $(MAKEPARALLEL)
+	rm -rf $(MAKEPARALLEL_INTERMEDIATES_PATH)/*.o
+	rm -rf $(MAKEPARALLEL_INTERMEDIATES_PATH)/*.d
+
+.PHONY: makeparallel_clean
+
+-include $(MAKEPARALLEL_INTERMEDIATES_PATH)/*.d
+
+.PHONY: test
+test: $(MAKEPARALLEL)
+	MAKEFLAGS= $(MAKE) -j1234 -C $(MAKEPARALLEL_SRC_PATH) -f Makefile.test MAKEPARALLEL=$(MAKEPARALLEL) test

tools/makeparallel/Makefile.test

@@ -0,0 +1,5 @@
+MAKEPARALLEL ?= ./makeparallel
+
+.PHONY: test
+test:
+	+if [ "$$($(MAKEPARALLEL) echo)" = "-j1234" ]; then echo SUCCESS; else echo FAILED; fi

tools/makeparallel/README.md

@@ -0,0 +1,54 @@
+<!---
+Copyright (C) 2015 The Android Open Source Project
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+makeparallel
+============
+makeparallel communicates with the [GNU make jobserver](http://make.mad-scientist.net/papers/jobserver-implementation/)
+in order claim all available jobs, and then passes the number of jobs
+claimed to a subprocess with `-j<jobs>`.
+
+The number of available jobs is determined by reading tokens from the jobserver
+until a read would block.  If the makeparallel rule is the only one running the
+number of jobs will be the total size of the jobserver pool, i.e. the value
+passed to make with `-j`.  Any jobs running in parallel with with the
+makeparellel rule will reduce the measured value, and thus reduce the
+parallelism available to the subprocess.
+
+To run a multi-thread or multi-process binary inside GNU make using
+makeparallel, add
+```Makefile
+	+makeparallel subprocess arguments
+```
+to a rule.  For example, to wrap ninja in make, use something like:
+```Makefile
+	+makeparallel ninja -f build.ninja
+```
+
+To determine the size of the jobserver pool, add
+```Makefile
+	+makeparallel echo > make.jobs
+```
+to a rule that is guarantee to run alone (i.e. all other rules are either
+dependencies of the makeparallel rule, or the depend on the makeparallel
+rule.  The output file will contain the `-j<num>` flag passed to the parent
+make process, or `-j1` if no flag was found.  Since GNU make will run
+makeparallel during the execution phase, after all variables have been
+set and evaluated, it is not possible to get the output of makeparallel
+into a make variable.  Instead, use a shell substitution to read the output
+file directly in a recipe.  For example:
+```Makefile
+	echo Make was started with $$(cat make.jobs)
+```

tools/makeparallel/makeparallel.cpp

@@ -0,0 +1,266 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// makeparallel communicates with the GNU make jobserver
+// (http://make.mad-scientist.net/papers/jobserver-implementation/)
+// in order claim all available jobs, and then passes the number of jobs
+// claimed to a subprocess with -j<jobs>.
+
+#include <errno.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include <string>
+#include <vector>
+
+#ifdef __linux__
+#include <error.h>
+#endif
+
+#ifdef __APPLE__
+#include <err.h>
+#define error(code, eval, fmt, ...) errc(eval, code, fmt, ##__VA_ARGS__)
+// Darwin does not interrupt syscalls by default.
+#define TEMP_FAILURE_RETRY(exp) (exp)
+#endif
+
+// Throw an error if fd is not valid.
+static void CheckFd(int fd) {
+  int ret = fcntl(fd, F_GETFD);
+  if (ret < 0) {
+    if (errno == EBADF) {
+      error(errno, 0, "no jobserver pipe, prefix recipe command with '+'");
+    } else {
+      error(errno, errno, "fnctl failed");
+    }
+  }
+}
+
+// Extract --jobserver-fds= argument from MAKEFLAGS environment variable.
+static int GetJobserver(int* in_fd, int* out_fd) {
+  const char* makeflags_env = getenv("MAKEFLAGS");
+  if (makeflags_env == nullptr) {
+    return false;
+  }
+
+  std::string makeflags = makeflags_env;
+
+  const std::string jobserver_fds_arg = "--jobserver-fds=";
+  size_t start = makeflags.find(jobserver_fds_arg);
+
+  if (start == std::string::npos) {
+    return false;
+  }
+
+  start += jobserver_fds_arg.size();
+
+  std::string::size_type end = makeflags.find(' ', start);
+
+  std::string::size_type len;
+  if (end == std::string::npos) {
+    len = std::string::npos;
+  } else {
+    len = end - start;
+  }
+
+  std::string jobserver_fds = makeflags.substr(start, len);
+
+  if (sscanf(jobserver_fds.c_str(), "%d,%d", in_fd, out_fd) != 2) {
+    return false;
+  }
+
+  CheckFd(*in_fd);
+  CheckFd(*out_fd);
+
+  return true;
+}
+
+// Read a single byte from fd, with timeout in milliseconds.  Returns true if
+// a byte was read, false on timeout.  Throws away the read value.
+// Non-reentrant, uses timer and signal handler global state, plus static
+// variable to communicate with signal handler.
+//
+// Uses a SIGALRM timer to fire a signal after timeout_ms that will interrupt
+// the read syscall if it hasn't yet completed.  If the timer fires before the
+// read the read could block forever, so read from a dup'd fd and close it from
+// the signal handler, which will cause the read to return EBADF if it occurs
+// after the signal.
+// The dup/read/close combo is very similar to the system described to avoid
+// a deadlock between SIGCHLD and read at
+// http://make.mad-scientist.net/papers/jobserver-implementation/
+static bool ReadByteTimeout(int fd, int timeout_ms) {
+  // global variable to communicate with the signal handler
+  static int dup_fd = -1;
+
+  // dup the fd so the signal handler can close it without losing the real one
+  dup_fd = dup(fd);
+  if (dup_fd < 0) {
+    error(errno, errno, "dup failed");
+  }
+
+  // set up a signal handler that closes dup_fd on SIGALRM
+  struct sigaction action = {};
+  action.sa_flags = SA_SIGINFO,
+  action.sa_sigaction = [](int, siginfo_t*, void*) {
+    close(dup_fd);
+  };
+  struct sigaction oldaction = {};
+  int ret = sigaction(SIGALRM, &action, &oldaction);
+  if (ret < 0) {
+    error(errno, errno, "sigaction failed");
+  }
+
+  // queue a SIGALRM after timeout_ms
+  const struct itimerval timeout = {{}, {0, timeout_ms * 1000}};
+  ret = setitimer(ITIMER_REAL, &timeout, NULL);
+  if (ret < 0) {
+    error(errno, errno, "setitimer failed");
+  }
+
+  // start the blocking read
+  char buf;
+  int read_ret = read(dup_fd, &buf, 1);
+  int read_errno = errno;
+
+  // cancel the alarm in case it hasn't fired yet
+  const struct itimerval cancel = {};
+  ret = setitimer(ITIMER_REAL, &cancel, NULL);
+  if (ret < 0) {
+    error(errno, errno, "reset setitimer failed");
+  }
+
+  // remove the signal handler
+  ret = sigaction(SIGALRM, &oldaction, NULL);
+  if (ret < 0) {
+    error(errno, errno, "reset sigaction failed");
+  }
+
+  // clean up the dup'd fd in case the signal never fired
+  close(dup_fd);
+  dup_fd = -1;
+
+  if (read_ret == 0) {
+    error(EXIT_FAILURE, 0, "EOF on jobserver pipe");
+  } else if (read_ret > 0) {
+    return true;
+  } else if (read_errno == EINTR || read_errno == EBADF) {
+    return false;
+  } else {
+    error(read_errno, read_errno, "read failed");
+  }
+  abort();
+}
+
+// Measure the size of the jobserver pool by reading from in_fd until it blocks
+static int GetJobserverTokens(int in_fd) {
+  int tokens = 0;
+  pollfd pollfds[] = {{in_fd, POLLIN, 0}};
+  int ret;
+  while ((ret = TEMP_FAILURE_RETRY(poll(pollfds, 1, 0))) != 0) {
+    if (ret < 0) {
+      error(errno, errno, "poll failed");
+    } else if (pollfds[0].revents != POLLIN) {
+      error(EXIT_FAILURE, 0, "unexpected event %d\n", pollfds[0].revents);
+    }
+
+    // There is probably a job token in the jobserver pipe.  There is a chance
+    // another process reads it first, which would cause a blocking read to
+    // block forever (or until another process put a token back in the pipe).
+    // The file descriptor can't be set to O_NONBLOCK as that would affect
+    // all users of the pipe, including the parent make process.
+    // ReadByteTimeout emulates a non-blocking read on a !O_NONBLOCK socket
+    // using a SIGALRM that fires after a short timeout.
+    bool got_token = ReadByteTimeout(in_fd, 10);
+    if (!got_token) {
+      // No more tokens
+      break;
+    } else {
+      tokens++;
+    }
+  }
+
+  // This process implicitly gets a token, so pool size is measured size + 1
+  return tokens;
+}
+
+// Return tokens to the jobserver pool.
+static void PutJobserverTokens(int out_fd, int tokens) {
+  // Return all the tokens to the pipe
+  char buf = '+';
+  for (int i = 0; i < tokens; i++) {
+    int ret = TEMP_FAILURE_RETRY(write(out_fd, &buf, 1));
+    if (ret < 0) {
+      error(errno, errno, "write failed");
+    } else if (ret == 0) {
+      error(EXIT_FAILURE, 0, "EOF on jobserver pipe");
+    }
+  }
+}
+
+int main(int argc, char* argv[]) {
+  int in_fd = -1;
+  int out_fd = -1;
+  int tokens = 0;
+
+  const char* path = argv[1];
+  std::vector<char*> args(&argv[1], &argv[argc]);
+
+  if (GetJobserver(&in_fd, &out_fd)) {
+    fcntl(in_fd, F_SETFD, FD_CLOEXEC);
+    fcntl(out_fd, F_SETFD, FD_CLOEXEC);
+
+    tokens = GetJobserverTokens(in_fd);
+  }
+
+  std::string jarg = "-j" + std::to_string(tokens + 1);
+  args.push_back(strdup(jarg.c_str()));
+  args.push_back(nullptr);
+
+  pid_t pid = fork();
+  if (pid < 0) {
+    error(errno, errno, "fork failed");
+  } else if (pid == 0) {
+    // child
+    int ret = execvp(path, args.data());
+    if (ret < 0) {
+      error(errno, errno, "exec failed");
+    }
+    abort();
+  }
+
+  // parent
+  siginfo_t status = {};
+  int exit_status = 0;
+  int ret = waitid(P_PID, pid, &status, WEXITED);
+  if (ret < 0) {
+    error(errno, errno, "waitpid failed");
+  } else if (status.si_code == CLD_EXITED) {
+    exit_status = status.si_status;
+  } else {
+    exit_status = -(status.si_status);
+  }
+
+  if (tokens > 0) {
+    PutJobserverTokens(out_fd, tokens);
+  }
+  exit(exit_status);
+}