From 4bf1be7ec870bdb828d7420f121b710ce4a07816 Mon Sep 17 00:00:00 2001 From: Chris Fries Date: Tue, 11 Apr 2017 09:40:40 -0500 Subject: [PATCH 1/3] fastboot: add AdbWinUsbApi as a required module fastboot uses AdbWinUsbApi on Windows, let's keep it required. Test: rm -rf out/host; mma fastboot and confirm AdbWinUsbApi is in out/ Bug: 36810152 Change-Id: Ica8b27cb1d0bca260f716dc61fdcea2ccc282623 --- fastboot/Android.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastboot/Android.mk b/fastboot/Android.mk index 5610cc089..2c578a90e 100644 --- a/fastboot/Android.mk +++ b/fastboot/Android.mk @@ -51,7 +51,7 @@ LOCAL_CFLAGS_darwin := -Wno-unused-parameter LOCAL_SRC_FILES_windows := usb_windows.cpp LOCAL_STATIC_LIBRARIES_windows := AdbWinApi -LOCAL_REQUIRED_MODULES_windows := AdbWinApi +LOCAL_REQUIRED_MODULES_windows := AdbWinApi AdbWinUsbApi LOCAL_LDLIBS_windows := -lws2_32 LOCAL_C_INCLUDES_windows := development/host/windows/usb/api From 6a99971096a9c5a6752f494db1d9b57d4ab77af2 Mon Sep 17 00:00:00 2001 From: Chris Fries Date: Tue, 4 Apr 2017 09:52:47 -0500 Subject: [PATCH 2/3] fastboot: Cap max size sent to libsparse This is required for large (>INT_MAX) sparse limit reported by the target. Also, patch up return chains of "int" that need to deal with sizes bigger than 2GB as well as return negative error codes. Test: -S works with large max-download-size Test: Flash 3GB system.img with max-download-size 2.5GB Bug: 36810152 Change-Id: I562a50eabd706bd5b97c71a1aef07c1ffd1a2e5c --- fastboot/engine.cpp | 4 ++-- fastboot/fastboot.cpp | 6 +++++- fastboot/fastboot.h | 4 ++-- fastboot/protocol.cpp | 21 +++++++++++---------- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/fastboot/engine.cpp b/fastboot/engine.cpp index 728dcb8e8..9f7d22657 100644 --- a/fastboot/engine.cpp +++ b/fastboot/engine.cpp @@ -328,11 +328,11 @@ void fb_queue_wait_for_disconnect(void) queue_action(OP_WAIT_FOR_DISCONNECT, ""); } -int fb_execute_queue(Transport* transport) +int64_t fb_execute_queue(Transport* transport) { Action *a; char resp[FB_RESPONSE_SZ+1]; - int status = 0; + int64_t status = 0; a = action_list; if (!a) diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp index 704dc43bf..cb8e5c026 100644 --- a/fastboot/fastboot.cpp +++ b/fastboot/fastboot.cpp @@ -78,6 +78,10 @@ static const char* product = nullptr; static const char* cmdline = nullptr; static unsigned short vendor_id = 0; static int long_listing = 0; +// Don't resparse files in too-big chunks. +// libsparse will support INT_MAX, but this results in large allocations, so +// let's keep it at 1GB to avoid memory pressure on the host. +static constexpr int64_t RESPARSE_LIMIT = 1 * 1024 * 1024 * 1024; static int64_t sparse_limit = -1; static int64_t target_sparse_limit = -1; @@ -789,7 +793,7 @@ static int64_t get_sparse_limit(Transport* transport, int64_t size) { } if (size > limit) { - return limit; + return std::min(limit, RESPARSE_LIMIT); } return 0; diff --git a/fastboot/fastboot.h b/fastboot/fastboot.h index 6699b6afe..b62a2d8d8 100644 --- a/fastboot/fastboot.h +++ b/fastboot/fastboot.h @@ -41,7 +41,7 @@ struct sparse_file; /* protocol.c - fastboot protocol */ int fb_command(Transport* transport, const char* cmd); int fb_command_response(Transport* transport, const char* cmd, char* response); -int fb_download_data(Transport* transport, const void* data, uint32_t size); +int64_t fb_download_data(Transport* transport, const void* data, uint32_t size); int fb_download_data_sparse(Transport* transport, struct sparse_file* s); const std::string fb_get_error(); @@ -64,7 +64,7 @@ void fb_queue_command(const char *cmd, const char *msg); void fb_queue_download(const char *name, void *data, uint32_t size); void fb_queue_notice(const char *notice); void fb_queue_wait_for_disconnect(void); -int fb_execute_queue(Transport* transport); +int64_t fb_execute_queue(Transport* transport); void fb_set_active(const char *slot); /* util stuff */ diff --git a/fastboot/protocol.cpp b/fastboot/protocol.cpp index bfa83b041..bf0479e1d 100644 --- a/fastboot/protocol.cpp +++ b/fastboot/protocol.cpp @@ -48,7 +48,7 @@ const std::string fb_get_error() { return g_error; } -static int check_response(Transport* transport, uint32_t size, char* response) { +static int64_t check_response(Transport* transport, uint32_t size, char* response) { char status[65]; while (true) { @@ -105,7 +105,7 @@ static int check_response(Transport* transport, uint32_t size, char* response) { return -1; } -static int _command_start(Transport* transport, const char* cmd, uint32_t size, char* response) { +static int64_t _command_start(Transport* transport, const char* cmd, uint32_t size, char* response) { size_t cmdsize = strlen(cmd); if (cmdsize > 64) { g_error = android::base::StringPrintf("command too large (%zu)", cmdsize); @@ -125,14 +125,14 @@ static int _command_start(Transport* transport, const char* cmd, uint32_t size, return check_response(transport, size, response); } -static int _command_data(Transport* transport, const void* data, uint32_t size) { - int r = transport->Write(data, size); +static int64_t _command_data(Transport* transport, const void* data, uint32_t size) { + int64_t r = transport->Write(data, size); if (r < 0) { g_error = android::base::StringPrintf("data transfer failure (%s)", strerror(errno)); transport->Close(); return -1; } - if (r != ((int) size)) { + if (r != static_cast(size)) { g_error = "data transfer failure (short transfer)"; transport->Close(); return -1; @@ -140,17 +140,17 @@ static int _command_data(Transport* transport, const void* data, uint32_t size) return r; } -static int _command_end(Transport* transport) { +static int64_t _command_end(Transport* transport) { return check_response(transport, 0, 0) < 0 ? -1 : 0; } -static int _command_send(Transport* transport, const char* cmd, const void* data, uint32_t size, +static int64_t _command_send(Transport* transport, const char* cmd, const void* data, uint32_t size, char* response) { if (size == 0) { return -1; } - int r = _command_start(transport, cmd, size, response); + int64_t r = _command_start(transport, cmd, size, response); if (r < 0) { return -1; } @@ -180,7 +180,7 @@ int fb_command_response(Transport* transport, const char* cmd, char* response) { return _command_send_no_data(transport, cmd, response); } -int fb_download_data(Transport* transport, const void* data, uint32_t size) { +int64_t fb_download_data(Transport* transport, const void* data, uint32_t size) { char cmd[64]; snprintf(cmd, sizeof(cmd), "download:%08x", size); return _command_send(transport, cmd, data, size, 0) < 0 ? -1 : 0; @@ -242,7 +242,8 @@ static int fb_download_data_sparse_write(void *priv, const void *data, int len) static int fb_download_data_sparse_flush(Transport* transport) { if (transport_buf_len > 0) { - if (_command_data(transport, transport_buf, transport_buf_len) != transport_buf_len) { + int64_t r = _command_data(transport, transport_buf, transport_buf_len); + if (r != static_cast(transport_buf_len)) { return -1; } transport_buf_len = 0; From 0ea946c007cf76a96d867ba3e96505f72639960e Mon Sep 17 00:00:00 2001 From: Chris Fries Date: Wed, 12 Apr 2017 10:25:57 -0500 Subject: [PATCH 3/3] fastboot: Support larger transfers during flash Adding methods to queue and download flashable images by fd instead of by pointer, so that we can deal with sending large (up to 4GB) files on windows and linux. This gets past limitations on linux to read more than 2GB from a file at a time, as well as memory limitations on win32, in order to download up to 4GB in a single transfer. Test: fastboot -w Test: "flash-all" from nexus factory images site (incl. fastboot -w update) Test: fastboot flash with large and small image, large and small max-download-size Test: Sanity check flashing on win32, darwin, linux. Test: Sanity check 3GB image download (with 3GB max-download-size) on win32, darwin, linux. Bug: 36810152 Change-Id: I528d739d344eb080d59d721dadf3b3b34d4b375e --- fastboot/engine.cpp | 27 ++++++++++++++++++++---- fastboot/fastboot.cpp | 33 ++++++++++++++++++++--------- fastboot/fastboot.h | 2 ++ fastboot/protocol.cpp | 49 +++++++++++++++++++++++++++++++++++++------ 4 files changed, 91 insertions(+), 20 deletions(-) diff --git a/fastboot/engine.cpp b/fastboot/engine.cpp index 9f7d22657..bf887c9e9 100644 --- a/fastboot/engine.cpp +++ b/fastboot/engine.cpp @@ -44,6 +44,7 @@ #define OP_NOTICE 4 #define OP_DOWNLOAD_SPARSE 5 #define OP_WAIT_FOR_DISCONNECT 6 +#define OP_DOWNLOAD_FD 7 typedef struct Action Action; @@ -56,6 +57,7 @@ struct Action { char cmd[CMD_SIZE]; const char* prod; void* data; + int fd; // The protocol only supports 32-bit sizes, so you'll have to break // anything larger into chunks. @@ -142,7 +144,20 @@ void fb_queue_erase(const char *ptn) a->msg = mkmsg("erasing '%s'", ptn); } -void fb_queue_flash(const char *ptn, void *data, unsigned sz) +void fb_queue_flash_fd(const char *ptn, int fd, uint32_t sz) +{ + Action *a; + + a = queue_action(OP_DOWNLOAD_FD, ""); + a->fd = fd; + a->size = sz; + a->msg = mkmsg("sending '%s' (%d KB)", ptn, sz / 1024); + + a = queue_action(OP_COMMAND, "flash:%s", ptn); + a->msg = mkmsg("writing '%s'", ptn); +} + +void fb_queue_flash(const char *ptn, void *data, uint32_t sz) { Action *a; @@ -155,7 +170,7 @@ void fb_queue_flash(const char *ptn, void *data, unsigned sz) a->msg = mkmsg("writing '%s'", ptn); } -void fb_queue_flash_sparse(const char* ptn, struct sparse_file* s, unsigned sz, size_t current, +void fb_queue_flash_sparse(const char* ptn, struct sparse_file* s, uint32_t sz, size_t current, size_t total) { Action *a; @@ -282,7 +297,7 @@ static int cb_save(Action* a, int status, const char* resp) { return 0; } -void fb_queue_query_save(const char *var, char *dest, unsigned dest_size) +void fb_queue_query_save(const char *var, char *dest, uint32_t dest_size) { Action *a; a = queue_action(OP_QUERY, "getvar:%s", var); @@ -309,7 +324,7 @@ void fb_queue_command(const char *cmd, const char *msg) a->msg = msg; } -void fb_queue_download(const char *name, void *data, unsigned size) +void fb_queue_download(const char *name, void *data, uint32_t size) { Action *a = queue_action(OP_DOWNLOAD, ""); a->data = data; @@ -351,6 +366,10 @@ int64_t fb_execute_queue(Transport* transport) status = fb_download_data(transport, a->data, a->size); status = a->func(a, status, status ? fb_get_error().c_str() : ""); if (status) break; + } else if (a->op == OP_DOWNLOAD_FD) { + status = fb_download_data_fd(transport, a->fd, a->size); + status = a->func(a, status, status ? fb_get_error().c_str() : ""); + if (status) break; } else if (a->op == OP_COMMAND) { status = fb_command(transport, a->cmd); status = a->func(a, status, status ? fb_get_error().c_str() : ""); diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp index cb8e5c026..3b524ac3c 100644 --- a/fastboot/fastboot.cpp +++ b/fastboot/fastboot.cpp @@ -55,6 +55,7 @@ #include #include #include +#include #include #include @@ -67,6 +68,8 @@ #include "udp.h" #include "usb.h" +using android::base::unique_fd; + #ifndef O_BINARY #define O_BINARY 0 #endif @@ -95,7 +98,7 @@ static unsigned tags_offset = 0x00000100; static const std::string convert_fbe_marker_filename("convert_fbe"); enum fb_buffer_type { - FB_BUFFER, + FB_BUFFER_FD, FB_BUFFER_SPARSE, }; @@ -103,6 +106,7 @@ struct fastboot_buffer { enum fb_buffer_type type; void* data; int64_t sz; + int fd; }; static struct { @@ -826,10 +830,9 @@ static bool load_buf_fd(Transport* transport, int fd, struct fastboot_buffer* bu buf->type = FB_BUFFER_SPARSE; buf->data = s; } else { - void* data = load_fd(fd, &sz); - if (data == nullptr) return -1; - buf->type = FB_BUFFER; - buf->data = data; + buf->type = FB_BUFFER_FD; + buf->data = nullptr; + buf->fd = fd; buf->sz = sz; } @@ -837,11 +840,22 @@ static bool load_buf_fd(Transport* transport, int fd, struct fastboot_buffer* bu } static bool load_buf(Transport* transport, const char* fname, struct fastboot_buffer* buf) { - int fd = open(fname, O_RDONLY | O_BINARY); + unique_fd fd(TEMP_FAILURE_RETRY(open(fname, O_RDONLY | O_BINARY))); + if (fd == -1) { return false; } - return load_buf_fd(transport, fd, buf); + + struct stat s; + if (fstat(fd, &s)) { + return false; + } + if (!S_ISREG(s.st_mode)) { + errno = S_ISDIR(s.st_mode) ? EISDIR : EINVAL; + return false; + } + + return load_buf_fd(transport, fd.release(), buf); } static void flash_buf(const char *pname, struct fastboot_buffer *buf) @@ -864,9 +878,8 @@ static void flash_buf(const char *pname, struct fastboot_buffer *buf) } break; } - - case FB_BUFFER: - fb_queue_flash(pname, buf->data, buf->sz); + case FB_BUFFER_FD: + fb_queue_flash_fd(pname, buf->fd, buf->sz); break; default: die("unknown buffer type: %d", buf->type); diff --git a/fastboot/fastboot.h b/fastboot/fastboot.h index b62a2d8d8..3f9527055 100644 --- a/fastboot/fastboot.h +++ b/fastboot/fastboot.h @@ -42,6 +42,7 @@ struct sparse_file; int fb_command(Transport* transport, const char* cmd); int fb_command_response(Transport* transport, const char* cmd, char* response); int64_t fb_download_data(Transport* transport, const void* data, uint32_t size); +int64_t fb_download_data_fd(Transport* transport, int fd, uint32_t size); int fb_download_data_sparse(Transport* transport, struct sparse_file* s); const std::string fb_get_error(); @@ -51,6 +52,7 @@ const std::string fb_get_error(); /* engine.c - high level command queue engine */ bool fb_getvar(Transport* transport, const std::string& key, std::string* value); void fb_queue_flash(const char *ptn, void *data, uint32_t sz); +void fb_queue_flash_fd(const char *ptn, int fd, uint32_t sz); void fb_queue_flash_sparse(const char* ptn, struct sparse_file* s, uint32_t sz, size_t current, size_t total); void fb_queue_erase(const char *ptn); diff --git a/fastboot/protocol.cpp b/fastboot/protocol.cpp index bf0479e1d..334f81ffb 100644 --- a/fastboot/protocol.cpp +++ b/fastboot/protocol.cpp @@ -38,6 +38,7 @@ #include #include +#include #include "fastboot.h" #include "transport.h" @@ -168,6 +169,39 @@ static int64_t _command_send(Transport* transport, const char* cmd, const void* return size; } +static int64_t _command_send_fd(Transport* transport, const char* cmd, int fd, uint32_t size, + char* response) { + static constexpr uint32_t MAX_MAP_SIZE = 512 * 1024 * 1024; + off64_t offset = 0; + uint32_t remaining = size; + + if (_command_start(transport, cmd, size, response) < 0) { + return -1; + } + + while (remaining) { + android::FileMap filemap; + size_t len = std::min(remaining, MAX_MAP_SIZE); + + if (!filemap.create(NULL, fd, offset, len, true)) { + return -1; + } + + if (_command_data(transport, filemap.getDataPtr(), len) < 0) { + return -1; + } + + remaining -= len; + offset += len; + } + + if (_command_end(transport) < 0) { + return -1; + } + + return size; +} + static int _command_send_no_data(Transport* transport, const char* cmd, char* response) { return _command_start(transport, cmd, 0, response); } @@ -181,9 +215,13 @@ int fb_command_response(Transport* transport, const char* cmd, char* response) { } int64_t fb_download_data(Transport* transport, const void* data, uint32_t size) { - char cmd[64]; - snprintf(cmd, sizeof(cmd), "download:%08x", size); - return _command_send(transport, cmd, data, size, 0) < 0 ? -1 : 0; + std::string cmd(android::base::StringPrintf("download:%08x", size)); + return _command_send(transport, cmd.c_str(), data, size, 0) < 0 ? -1 : 0; +} + +int64_t fb_download_data_fd(Transport* transport, int fd, uint32_t size) { + std::string cmd(android::base::StringPrintf("download:%08x", size)); + return _command_send_fd(transport, cmd.c_str(), fd, size, 0) < 0 ? -1 : 0; } #define TRANSPORT_BUF_SIZE 1024 @@ -257,9 +295,8 @@ int fb_download_data_sparse(Transport* transport, struct sparse_file* s) { return -1; } - char cmd[64]; - snprintf(cmd, sizeof(cmd), "download:%08x", size); - int r = _command_start(transport, cmd, size, 0); + std::string cmd(android::base::StringPrintf("download:%08x", size)); + int r = _command_start(transport, cmd.c_str(), size, 0); if (r < 0) { return -1; }