138 lines
4.7 KiB
Go
138 lines
4.7 KiB
Go
// Copyright 2020 Google Inc. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package build
|
|
|
|
// This file contains the functionality to upload data from one location to
|
|
// another.
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"android/soong/ui/metrics"
|
|
"github.com/golang/protobuf/proto"
|
|
|
|
upload_proto "android/soong/ui/metrics/upload_proto"
|
|
)
|
|
|
|
const (
|
|
// Used to generate a raw protobuf file that contains information
|
|
// of the list of metrics files from host to destination storage.
|
|
uploadPbFilename = ".uploader.pb"
|
|
)
|
|
|
|
var (
|
|
// For testing purpose.
|
|
tmpDir = ioutil.TempDir
|
|
)
|
|
|
|
// pruneMetricsFiles iterates the list of paths, checking if a path exist.
|
|
// If a path is a file, it is added to the return list. If the path is a
|
|
// directory, a recursive call is made to add the children files of the
|
|
// path.
|
|
func pruneMetricsFiles(paths []string) []string {
|
|
var metricsFiles []string
|
|
for _, p := range paths {
|
|
fi, err := os.Stat(p)
|
|
// Some paths passed may not exist. For example, build errors protobuf
|
|
// file may not exist since the build was successful.
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
if fi.IsDir() {
|
|
if l, err := ioutil.ReadDir(p); err == nil {
|
|
files := make([]string, 0, len(l))
|
|
for _, fi := range l {
|
|
files = append(files, filepath.Join(p, fi.Name()))
|
|
}
|
|
metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...)
|
|
}
|
|
} else {
|
|
metricsFiles = append(metricsFiles, p)
|
|
}
|
|
}
|
|
return metricsFiles
|
|
}
|
|
|
|
// UploadMetrics uploads a set of metrics files to a server for analysis. An
|
|
// uploader full path is specified in ANDROID_ENABLE_METRICS_UPLOAD environment
|
|
// variable in order to upload the set of metrics files. The metrics files are
|
|
// first copied to a temporary directory and the uploader is then executed in
|
|
// the background to allow the user/system to continue working. Soong communicates
|
|
// to the uploader through the upload_proto raw protobuf file.
|
|
func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) {
|
|
ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics")
|
|
defer ctx.EndTrace()
|
|
|
|
uploader := config.MetricsUploaderApp()
|
|
if uploader == "" {
|
|
// If the uploader path was not specified, no metrics shall be uploaded.
|
|
return
|
|
}
|
|
|
|
// Several of the files might be directories.
|
|
metricsFiles := pruneMetricsFiles(paths)
|
|
if len(metricsFiles) == 0 {
|
|
return
|
|
}
|
|
|
|
// The temporary directory cannot be deleted as the metrics uploader is started
|
|
// in the background and requires to exist until the operation is done. The
|
|
// uploader can delete the directory as it is specified in the upload proto.
|
|
tmpDir, err := tmpDir("", "upload_metrics")
|
|
if err != nil {
|
|
ctx.Fatalf("failed to create a temporary directory to store the list of metrics files: %v\n", err)
|
|
}
|
|
|
|
for i, src := range metricsFiles {
|
|
dst := filepath.Join(tmpDir, filepath.Base(src))
|
|
if _, err := copyFile(src, dst); err != nil {
|
|
ctx.Fatalf("failed to copy %q to %q: %v\n", src, dst, err)
|
|
}
|
|
metricsFiles[i] = dst
|
|
}
|
|
|
|
// For platform builds, the branch and target name is hardcoded to specific
|
|
// values for later extraction of the metrics in the data metrics pipeline.
|
|
data, err := proto.Marshal(&upload_proto.Upload{
|
|
CreationTimestampMs: proto.Uint64(uint64(buildStarted.UnixNano() / int64(time.Millisecond))),
|
|
CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))),
|
|
BranchName: proto.String("developer-metrics"),
|
|
TargetName: proto.String("platform-build-systems-metrics"),
|
|
MetricsFiles: metricsFiles,
|
|
DirectoriesToDelete: []string{tmpDir},
|
|
})
|
|
if err != nil {
|
|
ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err)
|
|
}
|
|
|
|
pbFile := filepath.Join(tmpDir, uploadPbFilename)
|
|
if err := ioutil.WriteFile(pbFile, data, 0644); err != nil {
|
|
ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err)
|
|
}
|
|
|
|
// Start the uploader in the background as it takes several milliseconds to start the uploader
|
|
// and prepare the metrics for upload. This affects small shell commands like "lunch".
|
|
cmd := Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile)
|
|
if simpleOutput {
|
|
cmd.RunOrFatal()
|
|
} else {
|
|
cmd.RunAndStreamOrFatal()
|
|
}
|
|
}
|