From 34a478f572e83cde57b5ce4418c0c980e14d2e66 Mon Sep 17 00:00:00 2001 From: Josh Gao Date: Wed, 7 Aug 2019 14:23:17 -0700 Subject: [PATCH] adb: add ls_v2. Add a 64-bit size/time variant of `adb ls`. Bug: http://b/122955521 Test: adb shell dd if=/dev/zero bs=1m count=8192 of=/data/local/tmp/big Test: adb pull /data/local/tmp/big Test: adb ls /data/local/tmp Change-Id: I6ff857239995bc7b5c5f8dfd65a36fad41e67d85 --- adb/client/file_sync_client.cpp | 71 +++++++++++++++++++---------- adb/daemon/file_sync_service.cpp | 78 ++++++++++++++++++++++++-------- adb/file_sync_protocol.h | 29 ++++++++++-- adb/transport.cpp | 2 + adb/transport.h | 1 + 5 files changed, 132 insertions(+), 49 deletions(-) diff --git a/adb/client/file_sync_client.cpp b/adb/client/file_sync_client.cpp index 1783d5526..2db1d8580 100644 --- a/adb/client/file_sync_client.cpp +++ b/adb/client/file_sync_client.cpp @@ -52,6 +52,8 @@ #include #include +typedef void(sync_ls_cb)(unsigned mode, uint64_t size, uint64_t time, const char* name); + struct syncsendbuf { unsigned id; unsigned size; @@ -210,6 +212,7 @@ class SyncConnection { Error("failed to get feature set: %s", error.c_str()); } else { have_stat_v2_ = CanUseFeature(features_, kFeatureStat2); + have_ls_v2_ = CanUseFeature(features_, kFeatureLs2); fd.reset(adb_connect("sync:", &error)); if (fd < 0) { Error("connect failed: %s", error.c_str()); @@ -372,6 +375,45 @@ class SyncConnection { return true; } + bool SendLs(const char* path) { + return SendRequest(have_ls_v2_ ? ID_LIST_V2 : ID_LIST_V1, path); + } + + private: + template + static bool FinishLsImpl(borrowed_fd fd, const std::function& callback) { + using dent_type = + std::conditional_t; + + while (true) { + dent_type dent; + if (!ReadFdExactly(fd, &dent, sizeof(dent))) return false; + + uint32_t expected_id = v2 ? ID_DENT_V2 : ID_DENT_V1; + if (dent.id == ID_DONE) return true; + if (dent.id != expected_id) return false; + + // Maximum length of a file name excluding null terminator (NAME_MAX) on Linux is 255. + char buf[256]; + size_t len = dent.namelen; + if (len > 255) return false; + + if (!ReadFdExactly(fd, buf, len)) return false; + buf[len] = 0; + + callback(dent.mode, dent.size, dent.mtime, buf); + } + } + + public: + bool FinishLs(const std::function& callback) { + if (have_ls_v2_) { + return FinishLsImpl(this->fd, callback); + } else { + return FinishLsImpl(this->fd, callback); + } + } + // Sending header, payload, and footer in a single write makes a huge // difference to "adb sync" performance. bool SendSmallFile(const char* path_and_mode, @@ -578,6 +620,7 @@ class SyncConnection { bool expect_done_; FeatureSet features_; bool have_stat_v2_; + bool have_ls_v2_; TransferLedger global_ledger_; TransferLedger current_ledger_; @@ -609,28 +652,9 @@ class SyncConnection { } }; -typedef void (sync_ls_cb)(unsigned mode, unsigned size, unsigned time, const char* name); - static bool sync_ls(SyncConnection& sc, const char* path, const std::function& func) { - if (!sc.SendRequest(ID_LIST, path)) return false; - - while (true) { - syncmsg msg; - if (!ReadFdExactly(sc.fd, &msg.dent, sizeof(msg.dent))) return false; - - if (msg.dent.id == ID_DONE) return true; - if (msg.dent.id != ID_DENT) return false; - - size_t len = msg.dent.namelen; - if (len > 256) return false; // TODO: resize buffer? continue? - - char buf[257]; - if (!ReadFdExactly(sc.fd, buf, len)) return false; - buf[len] = 0; - - func(msg.dent.mode, msg.dent.size, msg.dent.mtime, buf); - } + return sc.SendLs(path) && sc.FinishLs(func); } static bool sync_stat(SyncConnection& sc, const char* path, struct stat* st) { @@ -787,9 +811,8 @@ bool do_sync_ls(const char* path) { SyncConnection sc; if (!sc.IsValid()) return false; - return sync_ls(sc, path, [](unsigned mode, unsigned size, unsigned time, - const char* name) { - printf("%08x %08x %08x %s\n", mode, size, time, name); + return sync_ls(sc, path, [](unsigned mode, uint64_t size, uint64_t time, const char* name) { + printf("%08x %08" PRIx64 " %08" PRIx64 " %s\n", mode, size, time, name); }); } @@ -1052,7 +1075,7 @@ static bool remote_build_list(SyncConnection& sc, std::vector* file_li file_list->push_back(ci); // Put the files/dirs in rpath on the lists. - auto callback = [&](unsigned mode, unsigned size, unsigned time, const char* name) { + auto callback = [&](unsigned mode, uint64_t size, uint64_t time, const char* name) { if (IsDotOrDotDot(name)) { return; } diff --git a/adb/daemon/file_sync_service.cpp b/adb/daemon/file_sync_service.cpp index c6e156352..d6af7087d 100644 --- a/adb/daemon/file_sync_service.cpp +++ b/adb/daemon/file_sync_service.cpp @@ -174,40 +174,73 @@ static bool do_stat_v2(int s, uint32_t id, const char* path) { return WriteFdExactly(s, &msg.stat_v2, sizeof(msg.stat_v2)); } +template static bool do_list(int s, const char* path) { dirent* de; - syncmsg msg; - msg.dent.id = ID_DENT; + using MessageType = + std::conditional_t; + MessageType msg; + uint32_t msg_id; + if constexpr (v2) { + msg_id = ID_DENT_V2; + } else { + msg_id = ID_DENT_V1; + } std::unique_ptr d(opendir(path), closedir); if (!d) goto done; while ((de = readdir(d.get()))) { + memset(&msg, 0, sizeof(msg)); + msg.id = msg_id; + std::string filename(StringPrintf("%s/%s", path, de->d_name)); struct stat st; if (lstat(filename.c_str(), &st) == 0) { - size_t d_name_length = strlen(de->d_name); - msg.dent.mode = st.st_mode; - msg.dent.size = st.st_size; - msg.dent.mtime = st.st_mtime; - msg.dent.namelen = d_name_length; + msg.mode = st.st_mode; + msg.size = st.st_size; + msg.mtime = st.st_mtime; - if (!WriteFdExactly(s, &msg.dent, sizeof(msg.dent)) || - !WriteFdExactly(s, de->d_name, d_name_length)) { - return false; + if constexpr (v2) { + msg.dev = st.st_dev; + msg.ino = st.st_ino; + msg.nlink = st.st_nlink; + msg.uid = st.st_uid; + msg.gid = st.st_gid; + msg.atime = st.st_atime; + msg.ctime = st.st_ctime; } + } else { + if constexpr (v2) { + msg.error = errno; + } else { + continue; + } + } + + size_t d_name_length = strlen(de->d_name); + msg.namelen = d_name_length; + + if (!WriteFdExactly(s, &msg, sizeof(msg)) || + !WriteFdExactly(s, de->d_name, d_name_length)) { + return false; } } done: - msg.dent.id = ID_DONE; - msg.dent.mode = 0; - msg.dent.size = 0; - msg.dent.mtime = 0; - msg.dent.namelen = 0; - return WriteFdExactly(s, &msg.dent, sizeof(msg.dent)); + memset(&msg, 0, sizeof(msg)); + msg.id = ID_DONE; + return WriteFdExactly(s, &msg, sizeof(msg)); +} + +static bool do_list_v1(int s, const char* path) { + return do_list(s, path); +} + +static bool do_list_v2(int s, const char* path) { + return do_list(s, path); } // Make sure that SendFail from adb_io.cpp isn't accidentally used in this file. @@ -499,8 +532,10 @@ static const char* sync_id_to_name(uint32_t id) { return "lstat_v2"; case ID_STAT_V2: return "stat_v2"; - case ID_LIST: - return "list"; + case ID_LIST_V1: + return "list_v1"; + case ID_LIST_V2: + return "list_v2"; case ID_SEND: return "send"; case ID_RECV: @@ -546,8 +581,11 @@ static bool handle_sync_command(int fd, std::vector& buffer) { case ID_STAT_V2: if (!do_stat_v2(fd, request.id, name)) return false; break; - case ID_LIST: - if (!do_list(fd, name)) return false; + case ID_LIST_V1: + if (!do_list_v1(fd, name)) return false; + break; + case ID_LIST_V2: + if (!do_list_v2(fd, name)) return false; break; case ID_SEND: if (!do_send(fd, name, buffer)) return false; diff --git a/adb/file_sync_protocol.h b/adb/file_sync_protocol.h index cd18a4092..508c13851 100644 --- a/adb/file_sync_protocol.h +++ b/adb/file_sync_protocol.h @@ -21,10 +21,14 @@ #define ID_LSTAT_V1 MKID('S', 'T', 'A', 'T') #define ID_STAT_V2 MKID('S', 'T', 'A', '2') #define ID_LSTAT_V2 MKID('L', 'S', 'T', '2') -#define ID_LIST MKID('L', 'I', 'S', 'T') + +#define ID_LIST_V1 MKID('L', 'I', 'S', 'T') +#define ID_LIST_V2 MKID('L', 'I', 'S', '2') +#define ID_DENT_V1 MKID('D', 'E', 'N', 'T') +#define ID_DENT_V2 MKID('D', 'N', 'T', '2') + #define ID_SEND MKID('S', 'E', 'N', 'D') #define ID_RECV MKID('R', 'E', 'C', 'V') -#define ID_DENT MKID('D', 'E', 'N', 'T') #define ID_DONE MKID('D', 'O', 'N', 'E') #define ID_DATA MKID('D', 'A', 'T', 'A') #define ID_OKAY MKID('O', 'K', 'A', 'Y') @@ -64,15 +68,30 @@ union syncmsg { uint32_t size; uint32_t mtime; uint32_t namelen; - } dent; + } dent_v1; // followed by `namelen` bytes of the name. + struct __attribute__((packed)) { + uint32_t id; + uint32_t error; + uint64_t dev; + uint64_t ino; + uint32_t mode; + uint32_t nlink; + uint32_t uid; + uint32_t gid; + uint64_t size; + int64_t atime; + int64_t mtime; + int64_t ctime; + uint32_t namelen; + } dent_v2; // followed by `namelen` bytes of the name. struct __attribute__((packed)) { uint32_t id; uint32_t size; - } data; + } data; // followed by `size` bytes of data. struct __attribute__((packed)) { uint32_t id; uint32_t msglen; - } status; + } status; // followed by `msglen` bytes of error message, if id == ID_FAIL. }; #define SYNC_DATA_MAX (64 * 1024) diff --git a/adb/transport.cpp b/adb/transport.cpp index 3d1d620d2..bebd27c3c 100644 --- a/adb/transport.cpp +++ b/adb/transport.cpp @@ -66,6 +66,7 @@ static auto& transport_lock = *new std::recursive_mutex(); const char* const kFeatureShell2 = "shell_v2"; const char* const kFeatureCmd = "cmd"; const char* const kFeatureStat2 = "stat_v2"; +const char* const kFeatureLs2 = "ls_v2"; const char* const kFeatureLibusb = "libusb"; const char* const kFeaturePushSync = "push_sync"; const char* const kFeatureApex = "apex"; @@ -1044,6 +1045,7 @@ const FeatureSet& supported_features() { kFeatureShell2, kFeatureCmd, kFeatureStat2, + kFeatureLs2, kFeatureFixedPushMkdir, kFeatureApex, kFeatureAbb, diff --git a/adb/transport.h b/adb/transport.h index f4490eded..c4a68417c 100644 --- a/adb/transport.h +++ b/adb/transport.h @@ -57,6 +57,7 @@ extern const char* const kFeatureShell2; // The 'cmd' command is available extern const char* const kFeatureCmd; extern const char* const kFeatureStat2; +extern const char* const kFeatureLs2; // The server is running with libusb enabled. extern const char* const kFeatureLibusb; // adbd supports `push --sync`.