Merge changes Ifda7de48,Ie33505f9 into rvc-dev
* changes: [adb incremental] send priority blocks first [adb data server] wait for installation results before terminates
This commit is contained in:
commit
c2872d831d
|
@ -298,6 +298,7 @@ cc_binary_host {
|
|||
"client/fastdeploycallbacks.cpp",
|
||||
"client/incremental.cpp",
|
||||
"client/incremental_server.cpp",
|
||||
"client/incremental_utils.cpp",
|
||||
"shell_service_protocol.cpp",
|
||||
],
|
||||
|
||||
|
|
|
@ -1423,6 +1423,26 @@ static bool _is_valid_ack_reply_fd(const int ack_reply_fd) {
|
|||
#endif
|
||||
}
|
||||
|
||||
static bool _is_valid_fd(int fd) {
|
||||
// Disallow invalid FDs and stdin/out/err as well.
|
||||
if (fd < 3) {
|
||||
return false;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
HANDLE handle = adb_get_os_handle(fd);
|
||||
DWORD info = 0;
|
||||
if (GetHandleInformation(handle, &info) == 0) {
|
||||
return false;
|
||||
}
|
||||
#else
|
||||
int flags = fcntl(fd, F_GETFD);
|
||||
if (flags == -1) {
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
|
||||
int adb_commandline(int argc, const char** argv) {
|
||||
bool no_daemon = false;
|
||||
bool is_daemon = false;
|
||||
|
@ -1977,17 +1997,28 @@ int adb_commandline(int argc, const char** argv) {
|
|||
}
|
||||
}
|
||||
} else if (!strcmp(argv[0], "inc-server")) {
|
||||
if (argc < 3) {
|
||||
error_exit("usage: adb inc-server FD FILE1 FILE2 ...");
|
||||
if (argc < 4) {
|
||||
#ifdef _WIN32
|
||||
error_exit("usage: adb inc-server CONNECTION_HANDLE OUTPUT_HANDLE FILE1 FILE2 ...");
|
||||
#else
|
||||
error_exit("usage: adb inc-server CONNECTION_FD OUTPUT_FD FILE1 FILE2 ...");
|
||||
#endif
|
||||
}
|
||||
int fd = atoi(argv[1]);
|
||||
if (fd < 3) {
|
||||
// Disallow invalid FDs and stdin/out/err as well.
|
||||
error_exit("Invalid fd number given: %d", fd);
|
||||
int connection_fd = atoi(argv[1]);
|
||||
if (!_is_valid_fd(connection_fd)) {
|
||||
error_exit("Invalid connection_fd number given: %d", connection_fd);
|
||||
}
|
||||
fd = adb_register_socket(fd);
|
||||
close_on_exec(fd);
|
||||
return incremental::serve(fd, argc - 2, argv + 2);
|
||||
|
||||
connection_fd = adb_register_socket(connection_fd);
|
||||
close_on_exec(connection_fd);
|
||||
|
||||
int output_fd = atoi(argv[2]);
|
||||
if (!_is_valid_fd(output_fd)) {
|
||||
error_exit("Invalid output_fd number given: %d", output_fd);
|
||||
}
|
||||
output_fd = adb_register_socket(output_fd);
|
||||
close_on_exec(output_fd);
|
||||
return incremental::serve(connection_fd, output_fd, argc - 3, argv + 3);
|
||||
}
|
||||
|
||||
error_exit("unknown command %s", argv[0]);
|
||||
|
|
|
@ -193,20 +193,72 @@ std::optional<Process> install(std::vector<std::string> files) {
|
|||
auto fd_param = std::to_string(osh);
|
||||
#endif
|
||||
|
||||
// pipe for child process to write output
|
||||
int print_fds[2];
|
||||
if (adb_socketpair(print_fds) != 0) {
|
||||
fprintf(stderr, "Failed to create socket pair for child to print to parent\n");
|
||||
return {};
|
||||
}
|
||||
auto [pipe_read_fd, pipe_write_fd] = print_fds;
|
||||
auto pipe_write_fd_param = std::to_string(pipe_write_fd);
|
||||
close_on_exec(pipe_read_fd);
|
||||
|
||||
std::vector<std::string> args(std::move(files));
|
||||
args.insert(args.begin(), {"inc-server", fd_param});
|
||||
auto child = adb_launch_process(adb_path, std::move(args), {connection_fd.get()});
|
||||
args.insert(args.begin(), {"inc-server", fd_param, pipe_write_fd_param});
|
||||
auto child =
|
||||
adb_launch_process(adb_path, std::move(args), {connection_fd.get(), pipe_write_fd});
|
||||
if (!child) {
|
||||
fprintf(stderr, "adb: failed to fork: %s\n", strerror(errno));
|
||||
return {};
|
||||
}
|
||||
|
||||
adb_close(pipe_write_fd);
|
||||
|
||||
auto killOnExit = [](Process* p) { p->kill(); };
|
||||
std::unique_ptr<Process, decltype(killOnExit)> serverKiller(&child, killOnExit);
|
||||
// TODO: Terminate server process if installation fails.
|
||||
serverKiller.release();
|
||||
|
||||
Result result = wait_for_installation(pipe_read_fd);
|
||||
adb_close(pipe_read_fd);
|
||||
|
||||
if (result == Result::Success) {
|
||||
// adb client exits now but inc-server can continue
|
||||
serverKiller.release();
|
||||
}
|
||||
return child;
|
||||
}
|
||||
|
||||
Result wait_for_installation(int read_fd) {
|
||||
static constexpr int maxMessageSize = 256;
|
||||
std::vector<char> child_stdout(CHUNK_SIZE);
|
||||
int bytes_read;
|
||||
int buf_size = 0;
|
||||
// TODO(b/150865433): optimize child's output parsing
|
||||
while ((bytes_read = adb_read(read_fd, child_stdout.data() + buf_size,
|
||||
child_stdout.size() - buf_size)) > 0) {
|
||||
// print to parent's stdout
|
||||
fprintf(stdout, "%.*s", bytes_read, child_stdout.data() + buf_size);
|
||||
|
||||
buf_size += bytes_read;
|
||||
const std::string_view stdout_str(child_stdout.data(), buf_size);
|
||||
// wait till installation either succeeds or fails
|
||||
if (stdout_str.find("Success") != std::string::npos) {
|
||||
return Result::Success;
|
||||
}
|
||||
// on failure, wait for full message
|
||||
static constexpr auto failure_msg_head = "Failure ["sv;
|
||||
if (const auto begin_itr = stdout_str.find(failure_msg_head);
|
||||
begin_itr != std::string::npos) {
|
||||
if (buf_size >= maxMessageSize) {
|
||||
return Result::Failure;
|
||||
}
|
||||
const auto end_itr = stdout_str.rfind("]");
|
||||
if (end_itr != std::string::npos && end_itr >= begin_itr + failure_msg_head.size()) {
|
||||
return Result::Failure;
|
||||
}
|
||||
}
|
||||
child_stdout.resize(buf_size + CHUNK_SIZE);
|
||||
}
|
||||
return Result::None;
|
||||
}
|
||||
|
||||
} // namespace incremental
|
||||
|
|
|
@ -27,4 +27,7 @@ namespace incremental {
|
|||
|
||||
std::optional<Process> install(std::vector<std::string> files);
|
||||
|
||||
enum class Result { Success, Failure, None };
|
||||
Result wait_for_installation(int read_fd);
|
||||
|
||||
} // namespace incremental
|
||||
|
|
|
@ -18,13 +18,6 @@
|
|||
|
||||
#include "incremental_server.h"
|
||||
|
||||
#include "adb.h"
|
||||
#include "adb_io.h"
|
||||
#include "adb_trace.h"
|
||||
#include "adb_unique_fd.h"
|
||||
#include "adb_utils.h"
|
||||
#include "sysdeps.h"
|
||||
|
||||
#include <android-base/endian.h>
|
||||
#include <android-base/strings.h>
|
||||
#include <inttypes.h>
|
||||
|
@ -41,6 +34,14 @@
|
|||
#include <type_traits>
|
||||
#include <unordered_set>
|
||||
|
||||
#include "adb.h"
|
||||
#include "adb_io.h"
|
||||
#include "adb_trace.h"
|
||||
#include "adb_unique_fd.h"
|
||||
#include "adb_utils.h"
|
||||
#include "incremental_utils.h"
|
||||
#include "sysdeps.h"
|
||||
|
||||
namespace incremental {
|
||||
|
||||
static constexpr int kBlockSize = 4096;
|
||||
|
@ -49,6 +50,7 @@ static constexpr short kCompressionNone = 0;
|
|||
static constexpr short kCompressionLZ4 = 1;
|
||||
static constexpr int kCompressBound = std::max(kBlockSize, LZ4_COMPRESSBOUND(kBlockSize));
|
||||
static constexpr auto kReadBufferSize = 128 * 1024;
|
||||
static constexpr int kPollTimeoutMillis = 300000; // 5 minutes
|
||||
|
||||
using BlockSize = int16_t;
|
||||
using FileId = int16_t;
|
||||
|
@ -61,9 +63,10 @@ using MagicType = uint32_t;
|
|||
|
||||
static constexpr MagicType INCR = 0x494e4352; // LE INCR
|
||||
|
||||
static constexpr RequestType EXIT = 0;
|
||||
static constexpr RequestType SERVING_COMPLETE = 0;
|
||||
static constexpr RequestType BLOCK_MISSING = 1;
|
||||
static constexpr RequestType PREFETCH = 2;
|
||||
static constexpr RequestType DESTROY = 3;
|
||||
|
||||
static constexpr inline int64_t roundDownToBlockOffset(int64_t val) {
|
||||
return val & ~(kBlockSize - 1);
|
||||
|
@ -134,6 +137,7 @@ class File {
|
|||
// Plain file
|
||||
File(const char* filepath, FileId id, int64_t size, unique_fd fd) : File(filepath, id, size) {
|
||||
this->fd_ = std::move(fd);
|
||||
priority_blocks_ = PriorityBlocksForFile(filepath, fd_.get(), size);
|
||||
}
|
||||
int64_t ReadBlock(BlockIdx block_idx, void* buf, bool* is_zip_compressed,
|
||||
std::string* error) const {
|
||||
|
@ -145,6 +149,7 @@ class File {
|
|||
}
|
||||
|
||||
const unique_fd& RawFd() const { return fd_; }
|
||||
const std::vector<BlockIdx>& PriorityBlocks() const { return priority_blocks_; }
|
||||
|
||||
std::vector<bool> sentBlocks;
|
||||
NumBlocks sentBlocksCount = 0;
|
||||
|
@ -158,12 +163,13 @@ class File {
|
|||
sentBlocks.resize(numBytesToNumBlocks(size));
|
||||
}
|
||||
unique_fd fd_;
|
||||
std::vector<BlockIdx> priority_blocks_;
|
||||
};
|
||||
|
||||
class IncrementalServer {
|
||||
public:
|
||||
IncrementalServer(unique_fd fd, std::vector<File> files)
|
||||
: adb_fd_(std::move(fd)), files_(std::move(files)) {
|
||||
IncrementalServer(unique_fd adb_fd, unique_fd output_fd, std::vector<File> files)
|
||||
: adb_fd_(std::move(adb_fd)), output_fd_(std::move(output_fd)), files_(std::move(files)) {
|
||||
buffer_.reserve(kReadBufferSize);
|
||||
}
|
||||
|
||||
|
@ -174,14 +180,23 @@ class IncrementalServer {
|
|||
const File* file;
|
||||
BlockIdx overallIndex = 0;
|
||||
BlockIdx overallEnd = 0;
|
||||
BlockIdx priorityIndex = 0;
|
||||
|
||||
PrefetchState(const File& f) : file(&f), overallEnd((BlockIdx)f.sentBlocks.size()) {}
|
||||
PrefetchState(const File& f, BlockIdx start, int count)
|
||||
explicit PrefetchState(const File& f, BlockIdx start, int count)
|
||||
: file(&f),
|
||||
overallIndex(start),
|
||||
overallEnd(std::min<BlockIdx>(start + count, f.sentBlocks.size())) {}
|
||||
|
||||
bool done() const { return overallIndex >= overallEnd; }
|
||||
explicit PrefetchState(const File& f)
|
||||
: PrefetchState(f, 0, (BlockIdx)f.sentBlocks.size()) {}
|
||||
|
||||
bool done() const {
|
||||
const bool overallSent = (overallIndex >= overallEnd);
|
||||
if (file->PriorityBlocks().empty()) {
|
||||
return overallSent;
|
||||
}
|
||||
return overallSent && (priorityIndex >= (BlockIdx)file->PriorityBlocks().size());
|
||||
}
|
||||
};
|
||||
|
||||
bool SkipToRequest(void* buffer, size_t* size, bool blocking);
|
||||
|
@ -197,9 +212,10 @@ class IncrementalServer {
|
|||
void Send(const void* data, size_t size, bool flush);
|
||||
void Flush();
|
||||
using TimePoint = decltype(std::chrono::high_resolution_clock::now());
|
||||
bool Exit(std::optional<TimePoint> startTime, int missesCount, int missesSent);
|
||||
bool ServingComplete(std::optional<TimePoint> startTime, int missesCount, int missesSent);
|
||||
|
||||
unique_fd const adb_fd_;
|
||||
unique_fd const output_fd_;
|
||||
std::vector<File> files_;
|
||||
|
||||
// Incoming data buffer.
|
||||
|
@ -210,6 +226,9 @@ class IncrementalServer {
|
|||
long long sentSize_ = 0;
|
||||
|
||||
std::vector<char> pendingBlocks_;
|
||||
|
||||
// True when client notifies that all the data has been received
|
||||
bool servingComplete_;
|
||||
};
|
||||
|
||||
bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) {
|
||||
|
@ -217,7 +236,8 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking)
|
|||
// Looking for INCR magic.
|
||||
bool magic_found = false;
|
||||
int bcur = 0;
|
||||
for (int bsize = buffer_.size(); bcur + 4 < bsize; ++bcur) {
|
||||
int bsize = buffer_.size();
|
||||
for (bcur = 0; bcur + 4 < bsize; ++bcur) {
|
||||
uint32_t magic = be32toh(*(uint32_t*)(buffer_.data() + bcur));
|
||||
if (magic == INCR) {
|
||||
magic_found = true;
|
||||
|
@ -226,8 +246,8 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking)
|
|||
}
|
||||
|
||||
if (bcur > 0) {
|
||||
// Stream the rest to stderr.
|
||||
fprintf(stderr, "%.*s", bcur, buffer_.data());
|
||||
// output the rest.
|
||||
WriteFdExactly(output_fd_, buffer_.data(), bcur);
|
||||
erase_buffer_head(bcur);
|
||||
}
|
||||
|
||||
|
@ -239,17 +259,26 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking)
|
|||
}
|
||||
|
||||
adb_pollfd pfd = {adb_fd_.get(), POLLIN, 0};
|
||||
auto res = adb_poll(&pfd, 1, blocking ? -1 : 0);
|
||||
auto res = adb_poll(&pfd, 1, blocking ? kPollTimeoutMillis : 0);
|
||||
|
||||
if (res != 1) {
|
||||
WriteFdExactly(output_fd_, buffer_.data(), buffer_.size());
|
||||
if (res < 0) {
|
||||
fprintf(stderr, "Failed to poll: %s\n", strerror(errno));
|
||||
D("Failed to poll: %s\n", strerror(errno));
|
||||
return false;
|
||||
}
|
||||
if (blocking) {
|
||||
fprintf(stderr, "Timed out waiting for data from device.\n");
|
||||
}
|
||||
if (blocking && servingComplete_) {
|
||||
// timeout waiting from client. Serving is complete, so quit.
|
||||
return false;
|
||||
}
|
||||
*size = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
auto bsize = buffer_.size();
|
||||
bsize = buffer_.size();
|
||||
buffer_.resize(kReadBufferSize);
|
||||
int r = adb_read(adb_fd_, buffer_.data() + bsize, kReadBufferSize - bsize);
|
||||
if (r > 0) {
|
||||
|
@ -257,21 +286,19 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking)
|
|||
continue;
|
||||
}
|
||||
|
||||
if (r == -1) {
|
||||
fprintf(stderr, "Failed to read from fd %d: %d. Exit\n", adb_fd_.get(), errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
// socket is closed
|
||||
return false;
|
||||
D("Failed to read from fd %d: %d. Exit\n", adb_fd_.get(), errno);
|
||||
break;
|
||||
}
|
||||
// socket is closed. print remaining messages
|
||||
WriteFdExactly(output_fd_, buffer_.data(), buffer_.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
std::optional<RequestCommand> IncrementalServer::ReadRequest(bool blocking) {
|
||||
uint8_t commandBuf[sizeof(RequestCommand)];
|
||||
auto size = sizeof(commandBuf);
|
||||
if (!SkipToRequest(&commandBuf, &size, blocking)) {
|
||||
return {{EXIT}};
|
||||
return {{DESTROY}};
|
||||
}
|
||||
if (size < sizeof(RequestCommand)) {
|
||||
return {};
|
||||
|
@ -351,6 +378,17 @@ void IncrementalServer::RunPrefetching() {
|
|||
while (!prefetches_.empty() && blocksToSend > 0) {
|
||||
auto& prefetch = prefetches_.front();
|
||||
const auto& file = *prefetch.file;
|
||||
const auto& priority_blocks = file.PriorityBlocks();
|
||||
if (!priority_blocks.empty()) {
|
||||
for (auto& i = prefetch.priorityIndex;
|
||||
blocksToSend > 0 && i < (BlockIdx)priority_blocks.size(); ++i) {
|
||||
if (auto res = SendBlock(file.id, priority_blocks[i]); res == SendResult::Sent) {
|
||||
--blocksToSend;
|
||||
} else if (res == SendResult::Error) {
|
||||
fprintf(stderr, "Failed to send priority block %" PRId32 "\n", i);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto& i = prefetch.overallIndex; blocksToSend > 0 && i < prefetch.overallEnd; ++i) {
|
||||
if (auto res = SendBlock(file.id, i); res == SendResult::Sent) {
|
||||
--blocksToSend;
|
||||
|
@ -391,17 +429,17 @@ void IncrementalServer::Flush() {
|
|||
pendingBlocks_.clear();
|
||||
}
|
||||
|
||||
bool IncrementalServer::Exit(std::optional<TimePoint> startTime, int missesCount, int missesSent) {
|
||||
bool IncrementalServer::ServingComplete(std::optional<TimePoint> startTime, int missesCount,
|
||||
int missesSent) {
|
||||
servingComplete_ = true;
|
||||
using namespace std::chrono;
|
||||
auto endTime = high_resolution_clock::now();
|
||||
fprintf(stderr,
|
||||
"Connection failed or received exit command. Exit.\n"
|
||||
"Misses: %d, of those unique: %d; sent compressed: %d, uncompressed: "
|
||||
"%d, mb: %.3f\n"
|
||||
"Total time taken: %.3fms\n",
|
||||
missesCount, missesSent, compressed_, uncompressed_, sentSize_ / 1024.0 / 1024.0,
|
||||
duration_cast<microseconds>(endTime - (startTime ? *startTime : endTime)).count() /
|
||||
1000.0);
|
||||
D("Streaming completed.\n"
|
||||
"Misses: %d, of those unique: %d; sent compressed: %d, uncompressed: "
|
||||
"%d, mb: %.3f\n"
|
||||
"Total time taken: %.3fms\n",
|
||||
missesCount, missesSent, compressed_, uncompressed_, sentSize_ / 1024.0 / 1024.0,
|
||||
duration_cast<microseconds>(endTime - (startTime ? *startTime : endTime)).count() / 1000.0);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -425,7 +463,7 @@ bool IncrementalServer::Serve() {
|
|||
std::all_of(files_.begin(), files_.end(), [](const File& f) {
|
||||
return f.sentBlocksCount == NumBlocks(f.sentBlocks.size());
|
||||
})) {
|
||||
fprintf(stdout, "All files should be loaded. Notifying the device.\n");
|
||||
fprintf(stderr, "All files should be loaded. Notifying the device.\n");
|
||||
SendDone();
|
||||
doneSent = true;
|
||||
}
|
||||
|
@ -446,9 +484,14 @@ bool IncrementalServer::Serve() {
|
|||
BlockIdx blockIdx = request->block_idx;
|
||||
|
||||
switch (request->request_type) {
|
||||
case EXIT: {
|
||||
case DESTROY: {
|
||||
// Stop everything.
|
||||
return Exit(startTime, missesCount, missesSent);
|
||||
return true;
|
||||
}
|
||||
case SERVING_COMPLETE: {
|
||||
// Not stopping the server here.
|
||||
ServingComplete(startTime, missesCount, missesSent);
|
||||
break;
|
||||
}
|
||||
case BLOCK_MISSING: {
|
||||
++missesCount;
|
||||
|
@ -502,8 +545,9 @@ bool IncrementalServer::Serve() {
|
|||
}
|
||||
}
|
||||
|
||||
bool serve(int adb_fd, int argc, const char** argv) {
|
||||
auto connection_fd = unique_fd(adb_fd);
|
||||
bool serve(int connection_fd, int output_fd, int argc, const char** argv) {
|
||||
auto connection_ufd = unique_fd(connection_fd);
|
||||
auto output_ufd = unique_fd(output_fd);
|
||||
if (argc <= 0) {
|
||||
error_exit("inc-server: must specify at least one file.");
|
||||
}
|
||||
|
@ -526,7 +570,7 @@ bool serve(int adb_fd, int argc, const char** argv) {
|
|||
files.emplace_back(filepath, i, st.st_size, std::move(fd));
|
||||
}
|
||||
|
||||
IncrementalServer server(std::move(connection_fd), std::move(files));
|
||||
IncrementalServer server(std::move(connection_ufd), std::move(output_ufd), std::move(files));
|
||||
printf("Serving...\n");
|
||||
fclose(stdin);
|
||||
fclose(stdout);
|
||||
|
|
|
@ -21,6 +21,6 @@ namespace incremental {
|
|||
// Expecting arguments like:
|
||||
// {FILE1 FILE2 ...}
|
||||
// Where FILE* are files to serve.
|
||||
bool serve(int adbFd, int argc, const char** argv);
|
||||
bool serve(int connection_fd, int output_fd, int argc, const char** argv);
|
||||
|
||||
} // namespace incremental
|
||||
|
|
|
@ -0,0 +1,216 @@
|
|||
/*
|
||||
* Copyright (C) 2020 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#define TRACE_TAG INCREMENTAL
|
||||
|
||||
#include "incremental_utils.h"
|
||||
|
||||
#include <android-base/strings.h>
|
||||
#include <ziparchive/zip_archive.h>
|
||||
#include <ziparchive/zip_writer.h>
|
||||
|
||||
#include <cinttypes>
|
||||
#include <numeric>
|
||||
#include <unordered_set>
|
||||
|
||||
#include "sysdeps.h"
|
||||
|
||||
static constexpr int kBlockSize = 4096;
|
||||
|
||||
static constexpr inline int32_t offsetToBlockIndex(int64_t offset) {
|
||||
return (offset & ~(kBlockSize - 1)) >> 12;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
T valueAt(int fd, off64_t offset) {
|
||||
T t;
|
||||
memset(&t, 0, sizeof(T));
|
||||
if (adb_pread(fd, &t, sizeof(T), offset) != sizeof(T)) {
|
||||
memset(&t, -1, sizeof(T));
|
||||
}
|
||||
|
||||
return t;
|
||||
}
|
||||
|
||||
static void appendBlocks(int32_t start, int count, std::vector<int32_t>* blocks) {
|
||||
if (count == 1) {
|
||||
blocks->push_back(start);
|
||||
} else {
|
||||
auto oldSize = blocks->size();
|
||||
blocks->resize(oldSize + count);
|
||||
std::iota(blocks->begin() + oldSize, blocks->end(), start);
|
||||
}
|
||||
}
|
||||
|
||||
template <class T>
|
||||
static void unduplicate(std::vector<T>& v) {
|
||||
std::unordered_set<T> uniques(v.size());
|
||||
v.erase(std::remove_if(v.begin(), v.end(),
|
||||
[&uniques](T t) { return !uniques.insert(t).second; }),
|
||||
v.end());
|
||||
}
|
||||
|
||||
static off64_t CentralDirOffset(int fd, int64_t fileSize) {
|
||||
static constexpr int kZipEocdRecMinSize = 22;
|
||||
static constexpr int32_t kZipEocdRecSig = 0x06054b50;
|
||||
static constexpr int kZipEocdCentralDirSizeFieldOffset = 12;
|
||||
static constexpr int kZipEocdCommentLengthFieldOffset = 20;
|
||||
|
||||
int32_t sigBuf = 0;
|
||||
off64_t eocdOffset = -1;
|
||||
off64_t maxEocdOffset = fileSize - kZipEocdRecMinSize;
|
||||
int16_t commentLenBuf = 0;
|
||||
|
||||
// Search from the end of zip, backward to find beginning of EOCD
|
||||
for (int16_t commentLen = 0; commentLen < fileSize; ++commentLen) {
|
||||
sigBuf = valueAt<int32_t>(fd, maxEocdOffset - commentLen);
|
||||
if (sigBuf == kZipEocdRecSig) {
|
||||
commentLenBuf = valueAt<int16_t>(
|
||||
fd, maxEocdOffset - commentLen + kZipEocdCommentLengthFieldOffset);
|
||||
if (commentLenBuf == commentLen) {
|
||||
eocdOffset = maxEocdOffset - commentLen;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (eocdOffset < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
off64_t cdLen = static_cast<int64_t>(
|
||||
valueAt<int32_t>(fd, eocdOffset + kZipEocdCentralDirSizeFieldOffset));
|
||||
|
||||
return eocdOffset - cdLen;
|
||||
}
|
||||
|
||||
// Does not support APKs larger than 4GB
|
||||
static off64_t SignerBlockOffset(int fd, int64_t fileSize) {
|
||||
static constexpr int kApkSigBlockMinSize = 32;
|
||||
static constexpr int kApkSigBlockFooterSize = 24;
|
||||
static constexpr int64_t APK_SIG_BLOCK_MAGIC_HI = 0x3234206b636f6c42l;
|
||||
static constexpr int64_t APK_SIG_BLOCK_MAGIC_LO = 0x20676953204b5041l;
|
||||
|
||||
off64_t cdOffset = CentralDirOffset(fd, fileSize);
|
||||
if (cdOffset < 0) {
|
||||
return -1;
|
||||
}
|
||||
// CD offset is where original signer block ends. Search backwards for magic and footer.
|
||||
if (cdOffset < kApkSigBlockMinSize ||
|
||||
valueAt<int64_t>(fd, cdOffset - 2 * sizeof(int64_t)) != APK_SIG_BLOCK_MAGIC_LO ||
|
||||
valueAt<int64_t>(fd, cdOffset - sizeof(int64_t)) != APK_SIG_BLOCK_MAGIC_HI) {
|
||||
return -1;
|
||||
}
|
||||
int32_t signerSizeInFooter = valueAt<int32_t>(fd, cdOffset - kApkSigBlockFooterSize);
|
||||
off64_t signerBlockOffset = cdOffset - signerSizeInFooter - sizeof(int64_t);
|
||||
if (signerBlockOffset < 0) {
|
||||
return -1;
|
||||
}
|
||||
int32_t signerSizeInHeader = valueAt<int32_t>(fd, signerBlockOffset);
|
||||
if (signerSizeInFooter != signerSizeInHeader) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return signerBlockOffset;
|
||||
}
|
||||
|
||||
static std::vector<int32_t> ZipPriorityBlocks(off64_t signerBlockOffset, int64_t fileSize) {
|
||||
int32_t signerBlockIndex = offsetToBlockIndex(signerBlockOffset);
|
||||
int32_t lastBlockIndex = offsetToBlockIndex(fileSize);
|
||||
const auto numPriorityBlocks = lastBlockIndex - signerBlockIndex + 1;
|
||||
|
||||
std::vector<int32_t> zipPriorityBlocks;
|
||||
|
||||
// Some magic here: most of zip libraries perform a scan for EOCD record starting at the offset
|
||||
// of a maximum comment size from the end of the file. This means the last 65-ish KBs will be
|
||||
// accessed first, followed by the rest of the central directory blocks. Make sure we
|
||||
// send the data in the proper order, as central directory can be quite big by itself.
|
||||
static constexpr auto kMaxZipCommentSize = 64 * 1024;
|
||||
static constexpr auto kNumBlocksInEocdSearch = kMaxZipCommentSize / kBlockSize + 1;
|
||||
if (numPriorityBlocks > kNumBlocksInEocdSearch) {
|
||||
appendBlocks(lastBlockIndex - kNumBlocksInEocdSearch + 1, kNumBlocksInEocdSearch,
|
||||
&zipPriorityBlocks);
|
||||
appendBlocks(signerBlockIndex, numPriorityBlocks - kNumBlocksInEocdSearch,
|
||||
&zipPriorityBlocks);
|
||||
} else {
|
||||
appendBlocks(signerBlockIndex, numPriorityBlocks, &zipPriorityBlocks);
|
||||
}
|
||||
|
||||
// Somehow someone keeps accessing the start of the archive, even if there's nothing really
|
||||
// interesting there...
|
||||
appendBlocks(0, 1, &zipPriorityBlocks);
|
||||
return zipPriorityBlocks;
|
||||
}
|
||||
|
||||
// TODO(b/151676293): avoid using OpenArchiveFd that reads local file headers
|
||||
// which causes additional performance cost. Instead, only read from central directory.
|
||||
static std::vector<int32_t> InstallationPriorityBlocks(int fd, int64_t fileSize) {
|
||||
std::vector<int32_t> installationPriorityBlocks;
|
||||
ZipArchiveHandle zip;
|
||||
if (OpenArchiveFd(fd, "", &zip, false) != 0) {
|
||||
return {};
|
||||
}
|
||||
void* cookie = nullptr;
|
||||
if (StartIteration(zip, &cookie) != 0) {
|
||||
return {};
|
||||
}
|
||||
ZipEntry entry;
|
||||
std::string_view entryName;
|
||||
while (Next(cookie, &entry, &entryName) == 0) {
|
||||
if (entryName == "resources.arsc" || entryName == "AndroidManifest.xml" ||
|
||||
entryName.starts_with("lib/")) {
|
||||
// Full entries are needed for installation
|
||||
off64_t entryStartOffset = entry.offset;
|
||||
off64_t entryEndOffset =
|
||||
entryStartOffset +
|
||||
(entry.method == kCompressStored ? entry.uncompressed_length
|
||||
: entry.compressed_length) +
|
||||
(entry.has_data_descriptor ? 16 /* sizeof(DataDescriptor) */ : 0);
|
||||
int32_t startBlockIndex = offsetToBlockIndex(entryStartOffset);
|
||||
int32_t endBlockIndex = offsetToBlockIndex(entryEndOffset);
|
||||
int32_t numNewBlocks = endBlockIndex - startBlockIndex + 1;
|
||||
appendBlocks(startBlockIndex, numNewBlocks, &installationPriorityBlocks);
|
||||
} else if (entryName == "classes.dex") {
|
||||
// Only the head is needed for installation
|
||||
int32_t startBlockIndex = offsetToBlockIndex(entry.offset);
|
||||
appendBlocks(startBlockIndex, 1, &installationPriorityBlocks);
|
||||
}
|
||||
}
|
||||
|
||||
EndIteration(cookie);
|
||||
CloseArchive(zip);
|
||||
return installationPriorityBlocks;
|
||||
}
|
||||
|
||||
namespace incremental {
|
||||
std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, int fd, int64_t fileSize) {
|
||||
if (!android::base::EndsWithIgnoreCase(filepath, ".apk")) {
|
||||
return {};
|
||||
}
|
||||
off64_t signerOffset = SignerBlockOffset(fd, fileSize);
|
||||
if (signerOffset < 0) {
|
||||
// No signer block? not a valid APK
|
||||
return {};
|
||||
}
|
||||
std::vector<int32_t> priorityBlocks = ZipPriorityBlocks(signerOffset, fileSize);
|
||||
std::vector<int32_t> installationPriorityBlocks = InstallationPriorityBlocks(fd, fileSize);
|
||||
|
||||
priorityBlocks.insert(priorityBlocks.end(), installationPriorityBlocks.begin(),
|
||||
installationPriorityBlocks.end());
|
||||
unduplicate(priorityBlocks);
|
||||
return priorityBlocks;
|
||||
}
|
||||
} // namespace incremental
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Copyright (C) 2020 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace incremental {
|
||||
std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, int fd, int64_t fileSize);
|
||||
} // namespace incremental
|
Loading…
Reference in New Issue