diff --git a/adb/Android.mk b/adb/Android.mk index 5d6c418aa..68408f39e 100644 --- a/adb/Android.mk +++ b/adb/Android.mk @@ -116,7 +116,7 @@ LOCAL_SANITIZE := $(adb_target_sanitize) # Even though we're building a static library (and thus there's no link step for # this to take effect), this adds the includes to our path. -LOCAL_STATIC_LIBRARIES := libcrypto_utils libcrypto libbase +LOCAL_STATIC_LIBRARIES := libcrypto_utils libcrypto libbase libasyncio include $(BUILD_STATIC_LIBRARY) @@ -362,6 +362,7 @@ LOCAL_SANITIZE := $(adb_target_sanitize) LOCAL_STRIP_MODULE := keep_symbols LOCAL_STATIC_LIBRARIES := \ libadbd \ + libasyncio \ libavb_user \ libbase \ libqemu_pipe \ diff --git a/adb/adb.h b/adb/adb.h index 88e13b69b..6a9897f63 100644 --- a/adb/adb.h +++ b/adb/adb.h @@ -31,8 +31,7 @@ #include "usb.h" constexpr size_t MAX_PAYLOAD_V1 = 4 * 1024; -constexpr size_t MAX_PAYLOAD_V2 = 256 * 1024; -constexpr size_t MAX_PAYLOAD = MAX_PAYLOAD_V2; +constexpr size_t MAX_PAYLOAD = 1024 * 1024; constexpr size_t LINUX_MAX_SOCKET_SIZE = 4194304; diff --git a/adb/daemon/usb.cpp b/adb/daemon/usb.cpp index 0f92282c4..87ed3db67 100644 --- a/adb/daemon/usb.cpp +++ b/adb/daemon/usb.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -49,16 +50,11 @@ using namespace std::chrono_literals; #define MAX_PACKET_SIZE_HS 512 #define MAX_PACKET_SIZE_SS 1024 -// Kernels before 3.3 have a 16KiB transfer limit That limit was replaced -// with a 16MiB global limit in 3.3, but each URB submitted required a -// contiguous kernel allocation, so you would get ENOMEM if you tried to -// send something larger than the biggest available contiguous kernel -// memory region. Large contiguous allocations could be unreliable -// on a device kernel that has been running for a while fragmenting its -// memory so we start with a larger allocation, and shrink the amount if -// necessary. #define USB_FFS_BULK_SIZE 16384 +// Number of buffers needed to fit MAX_PAYLOAD, with an extra for ZLPs. +#define USB_FFS_NUM_BUFS ((MAX_PAYLOAD / USB_FFS_BULK_SIZE) + 1) + #define cpu_to_le16(x) htole16(x) #define cpu_to_le32(x) htole32(x) @@ -234,6 +230,26 @@ static const struct { }, }; +static void aio_block_init(aio_block* aiob) { + aiob->iocb.resize(USB_FFS_NUM_BUFS); + aiob->iocbs.resize(USB_FFS_NUM_BUFS); + aiob->events.resize(USB_FFS_NUM_BUFS); + aiob->num_submitted = 0; + for (unsigned i = 0; i < USB_FFS_NUM_BUFS; i++) { + aiob->iocbs[i] = &aiob->iocb[i]; + } +} + +static int getMaxPacketSize(int ffs_fd) { + usb_endpoint_descriptor desc; + if (ioctl(ffs_fd, FUNCTIONFS_ENDPOINT_DESC, reinterpret_cast(&desc))) { + D("[ could not get endpoint descriptor! (%d) ]", errno); + return MAX_PACKET_SIZE_HS; + } else { + return desc.wMaxPacketSize; + } +} + bool init_functionfs(struct usb_handle* h) { LOG(INFO) << "initializing functionfs"; @@ -301,6 +317,14 @@ bool init_functionfs(struct usb_handle* h) { goto err; } + if (io_setup(USB_FFS_NUM_BUFS, &h->read_aiob.ctx) || + io_setup(USB_FFS_NUM_BUFS, &h->write_aiob.ctx)) { + D("[ aio: got error on io_setup (%d) ]", errno); + } + + h->read_aiob.fd = h->bulk_out; + h->write_aiob.fd = h->bulk_in; + h->max_rw = MAX_PAYLOAD; while (h->max_rw >= USB_FFS_BULK_SIZE && retries < ENDPOINT_ALLOC_RETRIES) { int ret_in = ioctl(h->bulk_in, FUNCTIONFS_ENDPOINT_ALLOC, static_cast<__u32>(h->max_rw)); @@ -410,6 +434,65 @@ static int usb_ffs_read(usb_handle* h, void* data, int len) { return 0; } +static int usb_ffs_do_aio(usb_handle* h, const void* data, int len, bool read) { + aio_block* aiob = read ? &h->read_aiob : &h->write_aiob; + bool zero_packet = false; + + int num_bufs = len / USB_FFS_BULK_SIZE + (len % USB_FFS_BULK_SIZE == 0 ? 0 : 1); + const char* cur_data = reinterpret_cast(data); + int packet_size = getMaxPacketSize(aiob->fd); + + if (posix_madvise(const_cast(data), len, POSIX_MADV_SEQUENTIAL | POSIX_MADV_WILLNEED) < + 0) { + D("[ Failed to madvise: %d ]", errno); + } + + for (int i = 0; i < num_bufs; i++) { + int buf_len = std::min(len, USB_FFS_BULK_SIZE); + io_prep(&aiob->iocb[i], aiob->fd, cur_data, buf_len, 0, read); + + len -= buf_len; + cur_data += buf_len; + + if (len == 0 && buf_len % packet_size == 0 && read) { + // adb does not expect the device to send a zero packet after data transfer, + // but the host *does* send a zero packet for the device to read. + zero_packet = true; + } + } + if (zero_packet) { + io_prep(&aiob->iocb[num_bufs], aiob->fd, reinterpret_cast(cur_data), + packet_size, 0, read); + num_bufs += 1; + } + + if (io_submit(aiob->ctx, num_bufs, aiob->iocbs.data()) < num_bufs) { + D("[ aio: got error submitting %s (%d) ]", read ? "read" : "write", errno); + return -1; + } + if (TEMP_FAILURE_RETRY( + io_getevents(aiob->ctx, num_bufs, num_bufs, aiob->events.data(), nullptr)) < num_bufs) { + D("[ aio: got error waiting %s (%d) ]", read ? "read" : "write", errno); + return -1; + } + for (int i = 0; i < num_bufs; i++) { + if (aiob->events[i].res < 0) { + errno = aiob->events[i].res; + D("[ aio: got error event on %s (%d) ]", read ? "read" : "write", errno); + return -1; + } + } + return 0; +} + +static int usb_ffs_aio_read(usb_handle* h, void* data, int len) { + return usb_ffs_do_aio(h, data, len, true); +} + +static int usb_ffs_aio_write(usb_handle* h, const void* data, int len) { + return usb_ffs_do_aio(h, data, len, false); +} + static void usb_ffs_kick(usb_handle* h) { int err; @@ -438,6 +521,9 @@ static void usb_ffs_close(usb_handle* h) { h->kicked = false; adb_close(h->bulk_out); adb_close(h->bulk_in); + io_destroy(h->read_aiob.ctx); + io_destroy(h->write_aiob.ctx); + // Notify usb_adb_open_thread to open a new connection. h->lock.lock(); h->open_new_connection = true; @@ -450,8 +536,17 @@ static void usb_ffs_init() { usb_handle* h = new usb_handle(); - h->write = usb_ffs_write; - h->read = usb_ffs_read; + if (android::base::GetBoolProperty("sys.usb.ffs.aio_compat", false)) { + // Devices on older kernels (< 3.18) will not have aio support for ffs + // unless backported. Fall back on the non-aio functions instead. + h->write = usb_ffs_write; + h->read = usb_ffs_read; + } else { + h->write = usb_ffs_aio_write; + h->read = usb_ffs_aio_read; + aio_block_init(&h->read_aiob); + aio_block_init(&h->write_aiob); + } h->kick = usb_ffs_kick; h->close = usb_ffs_close; diff --git a/adb/daemon/usb.h b/adb/daemon/usb.h index 55b59952f..db1a6d6db 100644 --- a/adb/daemon/usb.h +++ b/adb/daemon/usb.h @@ -20,6 +20,17 @@ #include #include +#include + +struct aio_block { + std::vector iocb; + std::vector iocbs; + std::vector events; + aio_context_t ctx; + int num_submitted; + int fd; +}; + struct usb_handle { usb_handle() : kicked(false) { } @@ -39,7 +50,11 @@ struct usb_handle { int bulk_out = -1; /* "out" from the host's perspective => source for adbd */ int bulk_in = -1; /* "in" from the host's perspective => sink for adbd */ + // Access to these blocks is very not thread safe. Have one block for both the + // read and write threads. + struct aio_block read_aiob; + struct aio_block write_aiob; + int max_rw; }; -bool init_functionfs(struct usb_handle* h); diff --git a/adb/file_sync_client.cpp b/adb/file_sync_client.cpp index 2576fb15b..26f8d831c 100644 --- a/adb/file_sync_client.cpp +++ b/adb/file_sync_client.cpp @@ -441,7 +441,7 @@ class SyncConnection { syncsendbuf sbuf; sbuf.id = ID_DATA; while (true) { - int bytes_read = adb_read(lfd, sbuf.data, max); + int bytes_read = adb_read(lfd, sbuf.data, max - sizeof(SyncRequest)); if (bytes_read == -1) { Error("reading '%s' locally failed: %s", lpath, strerror(errno)); adb_close(lfd); diff --git a/adb/file_sync_service.cpp b/adb/file_sync_service.cpp index 3448ee053..c6f3e6691 100644 --- a/adb/file_sync_service.cpp +++ b/adb/file_sync_service.cpp @@ -206,6 +206,12 @@ static bool handle_send_file(int s, const char* path, uid_t uid, gid_t gid, uint __android_log_security_bswrite(SEC_TAG_ADB_SEND_FILE, path); int fd = adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode); + + if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED) < + 0) { + D("[ Failed to fadvise: %d ]", errno); + } + if (fd < 0 && errno == ENOENT) { if (!secure_mkdirs(android::base::Dirname(path))) { SendSyncFailErrno(s, "secure_mkdirs failed"); @@ -413,10 +419,14 @@ static bool do_recv(int s, const char* path, std::vector& buffer) { return false; } + if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE) < 0) { + D("[ Failed to fadvise: %d ]", errno); + } + syncmsg msg; msg.data.id = ID_DATA; while (true) { - int r = adb_read(fd, &buffer[0], buffer.size()); + int r = adb_read(fd, &buffer[0], buffer.size() - sizeof(msg.data)); if (r <= 0) { if (r == 0) break; SendSyncFailErrno(s, "read failed"); diff --git a/libasyncio/Android.bp b/libasyncio/Android.bp new file mode 100644 index 000000000..9a637acb7 --- /dev/null +++ b/libasyncio/Android.bp @@ -0,0 +1,44 @@ +// +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +libasyncio_cppflags = [ + "-Wall", + "-Wextra", + "-Werror", +] + +cc_library { + name: "libasyncio", + vendor_available: true, + host_supported: true, + srcs: [ + "AsyncIO.cpp", + ], + cppflags: libasyncio_cppflags, + + export_include_dirs: ["include"], + target: { + darwin: { + enabled: false, + }, + linux_bionic: { + enabled: true, + }, + windows: { + enabled: false, + }, + }, +} diff --git a/libasyncio/AsyncIO.cpp b/libasyncio/AsyncIO.cpp new file mode 100644 index 000000000..7430bc81e --- /dev/null +++ b/libasyncio/AsyncIO.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +int io_setup(unsigned nr, aio_context_t* ctxp) { + memset(ctxp, 0, sizeof(*ctxp)); + return syscall(__NR_io_setup, nr, ctxp); +} + +int io_destroy(aio_context_t ctx) { + return syscall(__NR_io_destroy, ctx); +} + +int io_submit(aio_context_t ctx, long nr, iocb** iocbpp) { + return syscall(__NR_io_submit, ctx, nr, iocbpp); +} + +int io_getevents(aio_context_t ctx, long min_nr, long max_nr, io_event* events, timespec* timeout) { + return syscall(__NR_io_getevents, ctx, min_nr, max_nr, events, timeout); +} + +int io_cancel(aio_context_t ctx, iocb* iocbp, io_event* result) { + return syscall(__NR_io_cancel, ctx, iocbp, result); +} + +void io_prep(iocb* iocb, int fd, const void* buf, uint64_t count, int64_t offset, bool read) { + memset(iocb, 0, sizeof(*iocb)); + iocb->aio_fildes = fd; + iocb->aio_lio_opcode = read ? IOCB_CMD_PREAD : IOCB_CMD_PWRITE; + iocb->aio_reqprio = 0; + iocb->aio_buf = reinterpret_cast(buf); + iocb->aio_nbytes = count; + iocb->aio_offset = offset; +} diff --git a/libasyncio/include/asyncio/AsyncIO.h b/libasyncio/include/asyncio/AsyncIO.h new file mode 100644 index 000000000..e3fb93a4d --- /dev/null +++ b/libasyncio/include/asyncio/AsyncIO.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _ASYNCIO_H +#define _ASYNCIO_H + +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Provides kernel aio operations. + */ + +int io_setup(unsigned nr, aio_context_t* ctxp); +int io_destroy(aio_context_t ctx); +int io_submit(aio_context_t ctx, long nr, iocb** iocbpp); +int io_getevents(aio_context_t ctx, long min_nr, long max_nr, io_event* events, timespec* timeout); +int io_cancel(aio_context_t ctx, iocb*, io_event* result); +void io_prep(iocb* iocb, int fd, const void* buf, uint64_t count, int64_t offset, bool read); + +#ifdef __cplusplus +}; +#endif + +#endif // ASYNCIO_H