diff --git a/adb/Android.bp b/adb/Android.bp index 12d9a1409..6386a78ed 100644 --- a/adb/Android.bp +++ b/adb/Android.bp @@ -470,6 +470,7 @@ cc_library { "libadbd_core", "libbrotli", "libdiagnose_usb", + "liblz4", ], shared_libs: [ @@ -571,6 +572,7 @@ cc_library { "libbrotli", "libcutils_sockets", "libdiagnose_usb", + "liblz4", "libmdnssd", ], @@ -605,6 +607,7 @@ cc_binary { "libadbd_services", "libasyncio", "libcap", + "liblz4", "libminijail", "libssl", ], diff --git a/adb/client/adb_install.cpp b/adb/client/adb_install.cpp index 092a86684..da3154e4a 100644 --- a/adb/client/adb_install.cpp +++ b/adb/client/adb_install.cpp @@ -290,7 +290,7 @@ static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy) } } - if (do_sync_push(apk_file, apk_dest.c_str(), false, true)) { + if (do_sync_push(apk_file, apk_dest.c_str(), false, CompressionType::Any, false)) { result = pm_command(argc, argv); delete_device_file(apk_dest); } diff --git a/adb/client/bugreport.cpp b/adb/client/bugreport.cpp index ab93f7d87..e162aaad1 100644 --- a/adb/client/bugreport.cpp +++ b/adb/client/bugreport.cpp @@ -282,5 +282,5 @@ int Bugreport::SendShellCommand(const std::string& command, bool disable_shell_p bool Bugreport::DoSyncPull(const std::vector& srcs, const char* dst, bool copy_attrs, const char* name) { - return do_sync_pull(srcs, dst, copy_attrs, false, name); + return do_sync_pull(srcs, dst, copy_attrs, CompressionType::None, name); } diff --git a/adb/client/commandline.cpp b/adb/client/commandline.cpp index 04b250df2..ceb21d595 100644 --- a/adb/client/commandline.cpp +++ b/adb/client/commandline.cpp @@ -129,20 +129,22 @@ static void help() { " reverse --remove-all remove all reverse socket connections from device\n" "\n" "file transfer:\n" - " push [--sync] [-zZ] LOCAL... REMOTE\n" + " push [--sync] [-z ALGORITHM] [-Z] LOCAL... REMOTE\n" " copy local files/directories to device\n" " --sync: only push files that are newer on the host than the device\n" - " -z: enable compression\n" + " -n: dry run: push files to device without storing to the filesystem\n" + " -z: enable compression with a specified algorithm (any, none, brotli)\n" " -Z: disable compression\n" - " pull [-azZ] REMOTE... LOCAL\n" + " pull [-a] [-z ALGORITHM] [-Z] REMOTE... LOCAL\n" " copy files/dirs from device\n" " -a: preserve file timestamp and mode\n" - " -z: enable compression\n" + " -z: enable compression with a specified algorithm (any, none, brotli)\n" " -Z: disable compression\n" - " sync [-lzZ] [all|data|odm|oem|product|system|system_ext|vendor]\n" + " sync [-l] [-z ALGORITHM] [-Z] [all|data|odm|oem|product|system|system_ext|vendor]\n" " sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n" + " -n: dry run: push files to device without storing to the filesystem\n" " -l: list files that would be copied, but don't copy them\n" - " -z: enable compression\n" + " -z: enable compression with a specified algorithm (any, none, brotli)\n" " -Z: disable compression\n" "\n" "shell:\n" @@ -1314,12 +1316,36 @@ static int restore(int argc, const char** argv) { return 0; } +static CompressionType parse_compression_type(const std::string& str, bool allow_numbers) { + if (allow_numbers) { + if (str == "0") { + return CompressionType::None; + } else if (str == "1") { + return CompressionType::Any; + } + } + + if (str == "any") { + return CompressionType::Any; + } else if (str == "none") { + return CompressionType::None; + } + + if (str == "brotli") { + return CompressionType::Brotli; + } else if (str == "lz4") { + return CompressionType::LZ4; + } + + error_exit("unexpected compression type %s", str.c_str()); +} + static void parse_push_pull_args(const char** arg, int narg, std::vector* srcs, - const char** dst, bool* copy_attrs, bool* sync, bool* compressed) { + const char** dst, bool* copy_attrs, bool* sync, + CompressionType* compression, bool* dry_run) { *copy_attrs = false; - const char* adb_compression = getenv("ADB_COMPRESSION"); - if (adb_compression && strcmp(adb_compression, "0") == 0) { - *compressed = false; + if (const char* adb_compression = getenv("ADB_COMPRESSION")) { + *compression = parse_compression_type(adb_compression, true); } srcs->clear(); @@ -1333,13 +1359,15 @@ static void parse_push_pull_args(const char** arg, int narg, std::vector srcs; const char* dst = nullptr; - parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compressed); + parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compression, + &dry_run); if (srcs.empty() || !dst) error_exit("push requires an argument"); - return do_sync_push(srcs, dst, sync, compressed) ? 0 : 1; + return do_sync_push(srcs, dst, sync, compression, dry_run) ? 0 : 1; } else if (!strcmp(argv[0], "pull")) { bool copy_attrs = false; - bool compressed = true; + CompressionType compression = CompressionType::Any; std::vector srcs; const char* dst = "."; - parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compressed); + parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compression, + nullptr); if (srcs.empty()) error_exit("pull requires an argument"); - return do_sync_pull(srcs, dst, copy_attrs, compressed) ? 0 : 1; + return do_sync_pull(srcs, dst, copy_attrs, compression) ? 0 : 1; } else if (!strcmp(argv[0], "install")) { if (argc < 2) error_exit("install requires an argument"); return install_app(argc, argv); @@ -1925,27 +1956,30 @@ int adb_commandline(int argc, const char** argv) { } else if (!strcmp(argv[0], "sync")) { std::string src; bool list_only = false; - bool compressed = true; + bool dry_run = false; + CompressionType compression = CompressionType::Any; - const char* adb_compression = getenv("ADB_COMPRESSION"); - if (adb_compression && strcmp(adb_compression, "0") == 0) { - compressed = false; + if (const char* adb_compression = getenv("ADB_COMPRESSION"); adb_compression) { + compression = parse_compression_type(adb_compression, true); } int opt; - while ((opt = getopt(argc, const_cast(argv), "lzZ")) != -1) { + while ((opt = getopt(argc, const_cast(argv), "lnz:Z")) != -1) { switch (opt) { case 'l': list_only = true; break; + case 'n': + dry_run = true; + break; case 'z': - compressed = true; + compression = parse_compression_type(optarg, false); break; case 'Z': - compressed = false; + compression = CompressionType::None; break; default: - error_exit("usage: adb sync [-lzZ] [PARTITION]"); + error_exit("usage: adb sync [-l] [-n] [-z ALGORITHM] [-Z] [PARTITION]"); } } @@ -1954,7 +1988,7 @@ int adb_commandline(int argc, const char** argv) { } else if (optind + 1 == argc) { src = argv[optind]; } else { - error_exit("usage: adb sync [-lzZ] [PARTITION]"); + error_exit("usage: adb sync [-l] [-n] [-z ALGORITHM] [-Z] [PARTITION]"); } std::vector partitions{"data", "odm", "oem", "product", @@ -1965,7 +1999,9 @@ int adb_commandline(int argc, const char** argv) { std::string src_dir{product_file(partition)}; if (!directory_exists(src_dir)) continue; found = true; - if (!do_sync_sync(src_dir, "/" + partition, list_only, compressed)) return 1; + if (!do_sync_sync(src_dir, "/" + partition, list_only, compression, dry_run)) { + return 1; + } } } if (!found) error_exit("don't know how to sync %s partition", src.c_str()); diff --git a/adb/client/fastdeploy.cpp b/adb/client/fastdeploy.cpp index de82e14e5..bc4b91bb9 100644 --- a/adb/client/fastdeploy.cpp +++ b/adb/client/fastdeploy.cpp @@ -112,7 +112,7 @@ static void push_to_device(const void* data, size_t byte_count, const char* dst, // but can't be removed until after the push. unix_close(tf.release()); - if (!do_sync_push(srcs, dst, sync, true)) { + if (!do_sync_push(srcs, dst, sync, CompressionType::Any, false)) { error_exit("Failed to push fastdeploy agent to device."); } } diff --git a/adb/client/file_sync_client.cpp b/adb/client/file_sync_client.cpp index 2ed58b2a3..681673449 100644 --- a/adb/client/file_sync_client.cpp +++ b/adb/client/file_sync_client.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include "sysdeps.h" @@ -236,6 +237,8 @@ class SyncConnection { have_ls_v2_ = CanUseFeature(features_, kFeatureLs2); have_sendrecv_v2_ = CanUseFeature(features_, kFeatureSendRecv2); have_sendrecv_v2_brotli_ = CanUseFeature(features_, kFeatureSendRecv2Brotli); + have_sendrecv_v2_lz4_ = CanUseFeature(features_, kFeatureSendRecv2LZ4); + have_sendrecv_v2_dry_run_send_ = CanUseFeature(features_, kFeatureSendRecv2DryRunSend); fd.reset(adb_connect("sync:", &error)); if (fd < 0) { Error("connect failed: %s", error.c_str()); @@ -261,6 +264,22 @@ class SyncConnection { bool HaveSendRecv2() const { return have_sendrecv_v2_; } bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; } + bool HaveSendRecv2LZ4() const { return have_sendrecv_v2_lz4_; } + bool HaveSendRecv2DryRunSend() const { return have_sendrecv_v2_dry_run_send_; } + + // Resolve a compression type which might be CompressionType::Any to a specific compression + // algorithm. + CompressionType ResolveCompressionType(CompressionType compression) const { + if (compression == CompressionType::Any) { + if (HaveSendRecv2LZ4()) { + return CompressionType::LZ4; + } else if (HaveSendRecv2Brotli()) { + return CompressionType::Brotli; + } + return CompressionType::None; + } + return compression; + } const FeatureSet& Features() const { return features_; } @@ -323,7 +342,7 @@ class SyncConnection { return WriteFdExactly(fd, buf.data(), buf.size()); } - bool SendSend2(std::string_view path, mode_t mode, bool compressed) { + bool SendSend2(std::string_view path, mode_t mode, CompressionType compression, bool dry_run) { if (path.length() > 1024) { Error("SendRequest failed: path too long: %zu", path.length()); errno = ENAMETOOLONG; @@ -339,7 +358,26 @@ class SyncConnection { syncmsg msg; msg.send_v2_setup.id = ID_SEND_V2; msg.send_v2_setup.mode = mode; - msg.send_v2_setup.flags = compressed ? kSyncFlagBrotli : kSyncFlagNone; + msg.send_v2_setup.flags = 0; + switch (compression) { + case CompressionType::None: + break; + + case CompressionType::Brotli: + msg.send_v2_setup.flags = kSyncFlagBrotli; + break; + + case CompressionType::LZ4: + msg.send_v2_setup.flags = kSyncFlagLZ4; + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + + if (dry_run) { + msg.send_v2_setup.flags |= kSyncFlagDryRun; + } buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.send_v2_setup)); @@ -352,7 +390,7 @@ class SyncConnection { return WriteFdExactly(fd, buf.data(), buf.size()); } - bool SendRecv2(const std::string& path) { + bool SendRecv2(const std::string& path, CompressionType compression) { if (path.length() > 1024) { Error("SendRequest failed: path too long: %zu", path.length()); errno = ENAMETOOLONG; @@ -367,7 +405,22 @@ class SyncConnection { syncmsg msg; msg.recv_v2_setup.id = ID_RECV_V2; - msg.recv_v2_setup.flags = kSyncFlagBrotli; + msg.recv_v2_setup.flags = 0; + switch (compression) { + case CompressionType::None: + break; + + case CompressionType::Brotli: + msg.recv_v2_setup.flags |= kSyncFlagBrotli; + break; + + case CompressionType::LZ4: + msg.recv_v2_setup.flags |= kSyncFlagLZ4; + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.recv_v2_setup)); @@ -494,7 +547,12 @@ class SyncConnection { // difference to "adb sync" performance. bool SendSmallFile(const std::string& path, mode_t mode, const std::string& lpath, const std::string& rpath, unsigned mtime, const char* data, - size_t data_length) { + size_t data_length, bool dry_run) { + if (dry_run) { + // We need to use send v2 for dry run. + return SendLargeFile(path, mode, lpath, rpath, mtime, CompressionType::None, dry_run); + } + std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode); if (path_and_mode.length() > 1024) { Error("SendSmallFile failed: path too long: %zu", path_and_mode.length()); @@ -533,9 +591,21 @@ class SyncConnection { return true; } - bool SendLargeFileCompressed(const std::string& path, mode_t mode, const std::string& lpath, - const std::string& rpath, unsigned mtime) { - if (!SendSend2(path, mode, true)) { + bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath, + const std::string& rpath, unsigned mtime, CompressionType compression, + bool dry_run) { + if (dry_run && !HaveSendRecv2DryRunSend()) { + Error("dry-run not supported by the device"); + return false; + } + + if (!HaveSendRecv2()) { + return SendLargeFileLegacy(path, mode, lpath, rpath, mtime); + } + + compression = ResolveCompressionType(compression); + + if (!SendSend2(path, mode, compression, dry_run)) { Error("failed to send ID_SEND_V2 message '%s': %s", path.c_str(), strerror(errno)); return false; } @@ -558,7 +628,25 @@ class SyncConnection { syncsendbuf sbuf; sbuf.id = ID_DATA; - BrotliEncoder encoder; + std::variant encoder_storage; + Encoder* encoder = nullptr; + switch (compression) { + case CompressionType::None: + encoder = &encoder_storage.emplace(SYNC_DATA_MAX); + break; + + case CompressionType::Brotli: + encoder = &encoder_storage.emplace(SYNC_DATA_MAX); + break; + + case CompressionType::LZ4: + encoder = &encoder_storage.emplace(SYNC_DATA_MAX); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + bool sending = true; while (sending) { Block input(SYNC_DATA_MAX); @@ -569,10 +657,10 @@ class SyncConnection { } if (r == 0) { - encoder.Finish(); + encoder->Finish(); } else { input.resize(r); - encoder.Append(std::move(input)); + encoder->Append(std::move(input)); RecordBytesTransferred(r); bytes_copied += r; ReportProgress(rpath, bytes_copied, total_size); @@ -580,7 +668,7 @@ class SyncConnection { while (true) { Block output; - EncodeResult result = encoder.Encode(&output); + EncodeResult result = encoder->Encode(&output); if (result == EncodeResult::Error) { Error("compressing '%s' locally failed", lpath.c_str()); return false; @@ -610,12 +698,8 @@ class SyncConnection { return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data)); } - bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath, - const std::string& rpath, unsigned mtime, bool compressed) { - if (compressed && HaveSendRecv2Brotli()) { - return SendLargeFileCompressed(path, mode, lpath, rpath, mtime); - } - + bool SendLargeFileLegacy(const std::string& path, mode_t mode, const std::string& lpath, + const std::string& rpath, unsigned mtime) { std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode); if (!SendRequest(ID_SEND_V1, path_and_mode)) { Error("failed to send ID_SEND_V1 message '%s': %s", path_and_mode.c_str(), @@ -840,6 +924,8 @@ class SyncConnection { bool have_ls_v2_; bool have_sendrecv_v2_; bool have_sendrecv_v2_brotli_; + bool have_sendrecv_v2_lz4_; + bool have_sendrecv_v2_dry_run_send_; TransferLedger global_ledger_; TransferLedger current_ledger_; @@ -921,7 +1007,8 @@ static bool sync_stat_fallback(SyncConnection& sc, const std::string& path, stru } static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::string& rpath, - unsigned mtime, mode_t mode, bool sync, bool compressed) { + unsigned mtime, mode_t mode, bool sync, CompressionType compression, + bool dry_run) { if (sync) { struct stat st; if (sync_lstat(sc, rpath, &st)) { @@ -942,7 +1029,7 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s } buf[data_length++] = '\0'; - if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length)) { + if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length, dry_run)) { return false; } return sc.ReadAcknowledgements(); @@ -960,11 +1047,12 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s sc.Error("failed to read all of '%s': %s", lpath.c_str(), strerror(errno)); return false; } - if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size())) { + if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size(), + dry_run)) { return false; } } else { - if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compressed)) { + if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compression, dry_run)) { return false; } } @@ -1027,8 +1115,10 @@ static bool sync_recv_v1(SyncConnection& sc, const char* rpath, const char* lpat } static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, - uint64_t expected_size) { - if (!sc.SendRecv2(rpath)) return false; + uint64_t expected_size, CompressionType compression) { + compression = sc.ResolveCompressionType(compression); + + if (!sc.SendRecv2(rpath, compression)) return false; adb_unlink(lpath); unique_fd lfd(adb_creat(lpath, 0644)); @@ -1040,9 +1130,28 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat uint64_t bytes_copied = 0; Block buffer(SYNC_DATA_MAX); - BrotliDecoder decoder(std::span(buffer.data(), buffer.size())); - bool reading = true; - while (reading) { + std::variant decoder_storage; + Decoder* decoder = nullptr; + + std::span buffer_span(buffer.data(), buffer.size()); + switch (compression) { + case CompressionType::None: + decoder = &decoder_storage.emplace(buffer_span); + break; + + case CompressionType::Brotli: + decoder = &decoder_storage.emplace(buffer_span); + break; + + case CompressionType::LZ4: + decoder = &decoder_storage.emplace(buffer_span); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + + while (true) { syncmsg msg; if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { adb_unlink(lpath); @@ -1050,33 +1159,32 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat } if (msg.data.id == ID_DONE) { - adb_unlink(lpath); - sc.Error("unexpected ID_DONE"); - return false; - } - - if (msg.data.id != ID_DATA) { + if (!decoder->Finish()) { + sc.Error("unexpected ID_DONE"); + return false; + } + } else if (msg.data.id != ID_DATA) { adb_unlink(lpath); sc.ReportCopyFailure(rpath, lpath, msg); return false; - } + } else { + if (msg.data.size > sc.max) { + sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); + adb_unlink(lpath); + return false; + } - if (msg.data.size > sc.max) { - sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); - adb_unlink(lpath); - return false; + Block block(msg.data.size); + if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) { + adb_unlink(lpath); + return false; + } + decoder->Append(std::move(block)); } - Block block(msg.data.size); - if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) { - adb_unlink(lpath); - return false; - } - decoder.Append(std::move(block)); - while (true) { std::span output; - DecodeResult result = decoder.Decode(&output); + DecodeResult result = decoder->Decode(&output); if (result == DecodeResult::Error) { sc.Error("decompress failed"); @@ -1093,8 +1201,7 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat } bytes_copied += output.size(); - - sc.RecordBytesTransferred(msg.data.size); + sc.RecordBytesTransferred(output.size()); sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size); if (result == DecodeResult::NeedInput) { @@ -1102,33 +1209,19 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat } else if (result == DecodeResult::MoreOutput) { continue; } else if (result == DecodeResult::Done) { - reading = false; - break; + sc.RecordFilesTransferred(1); + return true; } else { LOG(FATAL) << "invalid DecodeResult: " << static_cast(result); } } } - - syncmsg msg; - if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { - sc.Error("failed to read ID_DONE"); - return false; - } - - if (msg.data.id != ID_DONE) { - sc.Error("unexpected message after transfer: id = %d (expected ID_DONE)", msg.data.id); - return false; - } - - sc.RecordFilesTransferred(1); - return true; } static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, - uint64_t expected_size, bool compressed) { - if (sc.HaveSendRecv2() && compressed) { - return sync_recv_v2(sc, rpath, lpath, name, expected_size); + uint64_t expected_size, CompressionType compression) { + if (sc.HaveSendRecv2()) { + return sync_recv_v2(sc, rpath, lpath, name, expected_size, compression); } else { return sync_recv_v1(sc, rpath, lpath, name, expected_size); } @@ -1210,7 +1303,8 @@ static bool is_root_dir(std::string_view path) { } static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::string rpath, - bool check_timestamps, bool list_only, bool compressed) { + bool check_timestamps, bool list_only, + CompressionType compression, bool dry_run) { sc.NewTransfer(); // Make sure that both directory paths end in a slash. @@ -1292,7 +1386,8 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::st if (list_only) { sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str()); } else { - if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compressed)) { + if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compression, + dry_run)) { return false; } } @@ -1308,7 +1403,7 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::st } bool do_sync_push(const std::vector& srcs, const char* dst, bool sync, - bool compressed) { + CompressionType compression, bool dry_run) { SyncConnection sc; if (!sc.IsValid()) return false; @@ -1373,7 +1468,8 @@ bool do_sync_push(const std::vector& srcs, const char* dst, bool sy dst_dir.append(android::base::Basename(src_path)); } - success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compressed); + success &= + copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compression, dry_run); continue; } else if (!should_push_file(st.st_mode)) { sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode); @@ -1394,7 +1490,8 @@ bool do_sync_push(const std::vector& srcs, const char* dst, bool sy sc.NewTransfer(); sc.SetExpectedTotalBytes(st.st_size); - success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compressed); + success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compression, + dry_run); sc.ReportTransferRate(src_path, TransferDirection::push); } @@ -1480,7 +1577,7 @@ static int set_time_and_mode(const std::string& lpath, time_t time, } static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::string lpath, - bool copy_attrs, bool compressed) { + bool copy_attrs, CompressionType compression) { sc.NewTransfer(); // Make sure that both directory paths end in a slash. @@ -1510,7 +1607,7 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::st continue; } - if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compressed)) { + if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compression)) { return false; } @@ -1528,7 +1625,7 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::st } bool do_sync_pull(const std::vector& srcs, const char* dst, bool copy_attrs, - bool compressed, const char* name) { + CompressionType compression, const char* name) { SyncConnection sc; if (!sc.IsValid()) return false; @@ -1602,7 +1699,7 @@ bool do_sync_pull(const std::vector& srcs, const char* dst, bool co dst_dir.append(android::base::Basename(src_path)); } - success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compressed); + success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compression); continue; } else if (!should_pull_file(src_st.st_mode)) { sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode); @@ -1621,7 +1718,7 @@ bool do_sync_pull(const std::vector& srcs, const char* dst, bool co sc.NewTransfer(); sc.SetExpectedTotalBytes(src_st.st_size); - if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compressed)) { + if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compression)) { success = false; continue; } @@ -1638,11 +1735,11 @@ bool do_sync_pull(const std::vector& srcs, const char* dst, bool co } bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only, - bool compressed) { + CompressionType compression, bool dry_run) { SyncConnection sc; if (!sc.IsValid()) return false; - bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compressed); + bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compression, dry_run); if (!list_only) { sc.ReportOverallTransferRate(TransferDirection::push); } diff --git a/adb/client/file_sync_client.h b/adb/client/file_sync_client.h index de3f19245..cb8ca9323 100644 --- a/adb/client/file_sync_client.h +++ b/adb/client/file_sync_client.h @@ -19,11 +19,13 @@ #include #include +#include "file_sync_protocol.h" + bool do_sync_ls(const char* path); bool do_sync_push(const std::vector& srcs, const char* dst, bool sync, - bool compressed); + CompressionType compression, bool dry_run); bool do_sync_pull(const std::vector& srcs, const char* dst, bool copy_attrs, - bool compressed, const char* name = nullptr); + CompressionType compression, const char* name = nullptr); bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only, - bool compressed); + CompressionType compression, bool dry_run); diff --git a/adb/compression_utils.h b/adb/compression_utils.h index c445095ca..a0c48a207 100644 --- a/adb/compression_utils.h +++ b/adb/compression_utils.h @@ -16,10 +16,15 @@ #pragma once +#include +#include #include +#include + #include #include +#include #include "types.h" @@ -37,15 +42,103 @@ enum class EncodeResult { MoreOutput, }; -struct BrotliDecoder { +struct Decoder { + void Append(Block&& block) { input_buffer_.append(std::move(block)); } + bool Finish() { + bool old = std::exchange(finished_, true); + if (old) { + LOG(FATAL) << "Decoder::Finish called while already finished?"; + return false; + } + return true; + } + + virtual DecodeResult Decode(std::span* output) = 0; + + protected: + Decoder(std::span output_buffer) : output_buffer_(output_buffer) {} + ~Decoder() = default; + + bool finished_ = false; + IOVector input_buffer_; + std::span output_buffer_; +}; + +struct Encoder { + void Append(Block input) { input_buffer_.append(std::move(input)); } + bool Finish() { + bool old = std::exchange(finished_, true); + if (old) { + LOG(FATAL) << "Decoder::Finish called while already finished?"; + return false; + } + return true; + } + + virtual EncodeResult Encode(Block* output) = 0; + + protected: + explicit Encoder(size_t output_block_size) : output_block_size_(output_block_size) {} + ~Encoder() = default; + + const size_t output_block_size_; + bool finished_ = false; + IOVector input_buffer_; +}; + +struct NullDecoder final : public Decoder { + explicit NullDecoder(std::span output_buffer) : Decoder(output_buffer) {} + + DecodeResult Decode(std::span* output) final { + size_t available_out = output_buffer_.size(); + void* p = output_buffer_.data(); + while (available_out > 0 && !input_buffer_.empty()) { + size_t len = std::min(available_out, input_buffer_.front_size()); + p = mempcpy(p, input_buffer_.front_data(), len); + available_out -= len; + input_buffer_.drop_front(len); + } + *output = std::span(output_buffer_.data(), static_cast(p)); + if (input_buffer_.empty()) { + return finished_ ? DecodeResult::Done : DecodeResult::NeedInput; + } + return DecodeResult::MoreOutput; + } +}; + +struct NullEncoder final : public Encoder { + explicit NullEncoder(size_t output_block_size) : Encoder(output_block_size) {} + + EncodeResult Encode(Block* output) final { + output->clear(); + output->resize(output_block_size_); + + size_t available_out = output->size(); + void* p = output->data(); + + while (available_out > 0 && !input_buffer_.empty()) { + size_t len = std::min(available_out, input_buffer_.front_size()); + p = mempcpy(p, input_buffer_.front_data(), len); + available_out -= len; + input_buffer_.drop_front(len); + } + + output->resize(output->size() - available_out); + + if (input_buffer_.empty()) { + return finished_ ? EncodeResult::Done : EncodeResult::NeedInput; + } + return EncodeResult::MoreOutput; + } +}; + +struct BrotliDecoder final : public Decoder { explicit BrotliDecoder(std::span output_buffer) - : output_buffer_(output_buffer), + : Decoder(output_buffer), decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr), BrotliDecoderDestroyInstance) {} - void Append(Block&& block) { input_buffer_.append(std::move(block)); } - - DecodeResult Decode(std::span* output) { + DecodeResult Decode(std::span* output) final { size_t available_in = input_buffer_.front_size(); const uint8_t* next_in = reinterpret_cast(input_buffer_.front_data()); @@ -63,7 +156,8 @@ struct BrotliDecoder { switch (r) { case BROTLI_DECODER_RESULT_SUCCESS: - return DecodeResult::Done; + // We need to wait for ID_DONE from the other end. + return finished_ ? DecodeResult::Done : DecodeResult::NeedInput; case BROTLI_DECODER_RESULT_ERROR: return DecodeResult::Error; case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: @@ -77,33 +171,29 @@ struct BrotliDecoder { } private: - IOVector input_buffer_; - std::span output_buffer_; std::unique_ptr decoder_; }; -template -struct BrotliEncoder { - explicit BrotliEncoder() - : output_block_(OutputBlockSize), - output_bytes_left_(OutputBlockSize), +struct BrotliEncoder final : public Encoder { + explicit BrotliEncoder(size_t output_block_size) + : Encoder(output_block_size), + output_block_(output_block_size_), + output_bytes_left_(output_block_size_), encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr), BrotliEncoderDestroyInstance) { BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1); } - void Append(Block input) { input_buffer_.append(std::move(input)); } - void Finish() { finished_ = true; } - - EncodeResult Encode(Block* output) { + EncodeResult Encode(Block* output) final { output->clear(); + while (true) { size_t available_in = input_buffer_.front_size(); const uint8_t* next_in = reinterpret_cast(input_buffer_.front_data()); size_t available_out = output_bytes_left_; - uint8_t* next_out = reinterpret_cast(output_block_.data() + - (OutputBlockSize - output_bytes_left_)); + uint8_t* next_out = reinterpret_cast( + output_block_.data() + (output_block_size_ - output_bytes_left_)); BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS; if (finished_) { @@ -121,13 +211,13 @@ struct BrotliEncoder { output_bytes_left_ = available_out; if (BrotliEncoderIsFinished(encoder_.get())) { - output_block_.resize(OutputBlockSize - output_bytes_left_); + output_block_.resize(output_block_size_ - output_bytes_left_); *output = std::move(output_block_); return EncodeResult::Done; } else if (output_bytes_left_ == 0) { *output = std::move(output_block_); - output_block_.resize(OutputBlockSize); - output_bytes_left_ = OutputBlockSize; + output_block_.resize(output_block_size_); + output_bytes_left_ = output_block_size_; return EncodeResult::MoreOutput; } else if (input_buffer_.empty()) { return EncodeResult::NeedInput; @@ -136,9 +226,158 @@ struct BrotliEncoder { } private: - bool finished_ = false; - IOVector input_buffer_; Block output_block_; size_t output_bytes_left_; std::unique_ptr encoder_; }; + +struct LZ4Decoder final : public Decoder { + explicit LZ4Decoder(std::span output_buffer) + : Decoder(output_buffer), decoder_(nullptr, nullptr) { + LZ4F_dctx* dctx; + if (LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) != 0) { + LOG(FATAL) << "failed to initialize LZ4 decompression context"; + } + decoder_ = std::unique_ptr( + dctx, LZ4F_freeDecompressionContext); + } + + DecodeResult Decode(std::span* output) final { + size_t available_in = input_buffer_.front_size(); + const char* next_in = input_buffer_.front_data(); + + size_t available_out = output_buffer_.size(); + char* next_out = output_buffer_.data(); + + size_t rc = LZ4F_decompress(decoder_.get(), next_out, &available_out, next_in, + &available_in, nullptr); + if (LZ4F_isError(rc)) { + LOG(ERROR) << "LZ4F_decompress failed: " << LZ4F_getErrorName(rc); + return DecodeResult::Error; + } + + input_buffer_.drop_front(available_in); + + if (rc == 0) { + if (!input_buffer_.empty()) { + LOG(ERROR) << "LZ4 stream hit end before reading all data"; + return DecodeResult::Error; + } + lz4_done_ = true; + } + + *output = std::span(output_buffer_.data(), available_out); + + if (finished_) { + return input_buffer_.empty() && lz4_done_ ? DecodeResult::Done + : DecodeResult::MoreOutput; + } + + return DecodeResult::NeedInput; + } + + private: + bool lz4_done_ = false; + std::unique_ptr decoder_; +}; + +struct LZ4Encoder final : public Encoder { + explicit LZ4Encoder(size_t output_block_size) + : Encoder(output_block_size), encoder_(nullptr, nullptr) { + LZ4F_cctx* cctx; + if (LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) != 0) { + LOG(FATAL) << "failed to initialize LZ4 compression context"; + } + encoder_ = std::unique_ptr( + cctx, LZ4F_freeCompressionContext); + Block header(LZ4F_HEADER_SIZE_MAX); + size_t rc = LZ4F_compressBegin(encoder_.get(), header.data(), header.size(), nullptr); + if (LZ4F_isError(rc)) { + LOG(FATAL) << "LZ4F_compressBegin failed: %s", LZ4F_getErrorName(rc); + } + header.resize(rc); + output_buffer_.append(std::move(header)); + } + + // As an optimization, only emit a block if we have an entire output block ready, or we're done. + bool OutputReady() const { + return output_buffer_.size() >= output_block_size_ || lz4_finalized_; + } + + // TODO: Switch the output type to IOVector to remove a copy? + EncodeResult Encode(Block* output) final { + size_t available_in = input_buffer_.front_size(); + const char* next_in = input_buffer_.front_data(); + + // LZ4 makes no guarantees about being able to recover from trying to compress with an + // insufficiently large output buffer. LZ4F_compressBound tells us how much buffer we + // need to compress a given number of bytes, but the smallest value seems to be bigger + // than SYNC_DATA_MAX, so we need to buffer ourselves. + + // Input size chosen to be a local maximum for LZ4F_compressBound (i.e. the block size). + constexpr size_t max_input_size = 65536; + const size_t encode_block_size = LZ4F_compressBound(max_input_size, nullptr); + + if (available_in != 0) { + if (lz4_finalized_) { + LOG(ERROR) << "LZ4Encoder received data after Finish?"; + return EncodeResult::Error; + } + + available_in = std::min(available_in, max_input_size); + + Block encode_block(encode_block_size); + size_t available_out = encode_block.capacity(); + char* next_out = encode_block.data(); + + size_t rc = LZ4F_compressUpdate(encoder_.get(), next_out, available_out, next_in, + available_in, nullptr); + if (LZ4F_isError(rc)) { + LOG(ERROR) << "LZ4F_compressUpdate failed: " << LZ4F_getErrorName(rc); + return EncodeResult::Error; + } + + input_buffer_.drop_front(available_in); + + available_out -= rc; + next_out += rc; + + encode_block.resize(encode_block_size - available_out); + output_buffer_.append(std::move(encode_block)); + } + + if (finished_ && !lz4_finalized_) { + lz4_finalized_ = true; + + Block final_block(encode_block_size + 4); + size_t rc = LZ4F_compressEnd(encoder_.get(), final_block.data(), final_block.size(), + nullptr); + if (LZ4F_isError(rc)) { + LOG(ERROR) << "LZ4F_compressEnd failed: " << LZ4F_getErrorName(rc); + return EncodeResult::Error; + } + + final_block.resize(rc); + output_buffer_.append(std::move(final_block)); + } + + if (OutputReady()) { + size_t len = std::min(output_block_size_, output_buffer_.size()); + *output = output_buffer_.take_front(len).coalesce(); + } else { + output->clear(); + } + + if (lz4_finalized_ && output_buffer_.empty()) { + return EncodeResult::Done; + } else if (OutputReady()) { + return EncodeResult::MoreOutput; + } + return EncodeResult::NeedInput; + } + + private: + bool lz4_finalized_ = false; + std::unique_ptr encoder_; + IOVector output_buffer_; +}; diff --git a/adb/daemon/file_sync_service.cpp b/adb/daemon/file_sync_service.cpp index 5ccddeabf..d58131e5b 100644 --- a/adb/daemon/file_sync_service.cpp +++ b/adb/daemon/file_sync_service.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -266,37 +267,60 @@ static bool SendSyncFailErrno(borrowed_fd fd, const std::string& reason) { return SendSyncFail(fd, StringPrintf("%s: %s", reason.c_str(), strerror(errno))); } -static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp) { +static bool handle_send_file_data(borrowed_fd s, unique_fd fd, uint32_t* timestamp, + CompressionType compression) { syncmsg msg; - Block decode_buffer(SYNC_DATA_MAX); - BrotliDecoder decoder(std::span(decode_buffer.data(), decode_buffer.size())); + Block buffer(SYNC_DATA_MAX); + std::span buffer_span(buffer.data(), buffer.size()); + std::variant decoder_storage; + Decoder* decoder = nullptr; + + switch (compression) { + case CompressionType::None: + decoder = &decoder_storage.emplace(buffer_span); + break; + + case CompressionType::Brotli: + decoder = &decoder_storage.emplace(buffer_span); + break; + + case CompressionType::LZ4: + decoder = &decoder_storage.emplace(buffer_span); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + while (true) { if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false; - if (msg.data.id != ID_DATA) { - if (msg.data.id == ID_DONE) { - *timestamp = msg.data.size; - return true; - } + if (msg.data.id == ID_DONE) { + *timestamp = msg.data.size; + decoder->Finish(); + } else if (msg.data.id == ID_DATA) { + Block block(msg.data.size); + if (!ReadFdExactly(s, block.data(), msg.data.size)) return false; + decoder->Append(std::move(block)); + } else { SendSyncFail(s, "invalid data message"); return false; } - Block block(msg.data.size); - if (!ReadFdExactly(s, block.data(), msg.data.size)) return false; - decoder.Append(std::move(block)); - while (true) { std::span output; - DecodeResult result = decoder.Decode(&output); + DecodeResult result = decoder->Decode(&output); if (result == DecodeResult::Error) { SendSyncFailErrno(s, "decompress failed"); return false; } - if (!WriteFdExactly(fd, output.data(), output.size())) { - SendSyncFailErrno(s, "write failed"); - return false; + // fd is -1 if the client is pushing with --dry-run. + if (fd != -1) { + if (!WriteFdExactly(fd, output.data(), output.size())) { + SendSyncFailErrno(s, "write failed"); + return false; + } } if (result == DecodeResult::NeedInput) { @@ -304,7 +328,7 @@ static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* t } else if (result == DecodeResult::MoreOutput) { continue; } else if (result == DecodeResult::Done) { - break; + return true; } else { LOG(FATAL) << "invalid DecodeResult: " << static_cast(result); } @@ -314,102 +338,67 @@ static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* t __builtin_unreachable(); } -static bool handle_send_file_uncompressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp, - std::vector& buffer) { - syncmsg msg; - - while (true) { - if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false; - - if (msg.data.id != ID_DATA) { - if (msg.data.id == ID_DONE) { - *timestamp = msg.data.size; - return true; - } - SendSyncFail(s, "invalid data message"); - return false; - } - - if (msg.data.size > buffer.size()) { // TODO: resize buffer? - SendSyncFail(s, "oversize data message"); - return false; - } - if (!ReadFdExactly(s, &buffer[0], msg.data.size)) return false; - if (!WriteFdExactly(fd, &buffer[0], msg.data.size)) { - SendSyncFailErrno(s, "write failed"); - return false; - } - } -} - static bool handle_send_file(borrowed_fd s, const char* path, uint32_t* timestamp, uid_t uid, - gid_t gid, uint64_t capabilities, mode_t mode, bool compressed, - std::vector& buffer, bool do_unlink) { - int rc; + gid_t gid, uint64_t capabilities, mode_t mode, + CompressionType compression, bool dry_run, std::vector& buffer, + bool do_unlink) { syncmsg msg; + unique_fd fd; - __android_log_security_bswrite(SEC_TAG_ADB_SEND_FILE, path); - - unique_fd fd(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode)); - - if (fd < 0 && errno == ENOENT) { - if (!secure_mkdirs(Dirname(path))) { - SendSyncFailErrno(s, "secure_mkdirs failed"); - goto fail; - } + if (!dry_run) { + __android_log_security_bswrite(SEC_TAG_ADB_SEND_FILE, path); fd.reset(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode)); - } - if (fd < 0 && errno == EEXIST) { - fd.reset(adb_open_mode(path, O_WRONLY | O_CLOEXEC, mode)); - } - if (fd < 0) { - SendSyncFailErrno(s, "couldn't create file"); - goto fail; - } else { - if (fchown(fd.get(), uid, gid) == -1) { - SendSyncFailErrno(s, "fchown failed"); - goto fail; + + if (fd < 0 && errno == ENOENT) { + if (!secure_mkdirs(Dirname(path))) { + SendSyncFailErrno(s, "secure_mkdirs failed"); + goto fail; + } + fd.reset(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode)); } + if (fd < 0 && errno == EEXIST) { + fd.reset(adb_open_mode(path, O_WRONLY | O_CLOEXEC, mode)); + } + if (fd < 0) { + SendSyncFailErrno(s, "couldn't create file"); + goto fail; + } else { + if (fchown(fd.get(), uid, gid) == -1) { + SendSyncFailErrno(s, "fchown failed"); + goto fail; + } #if defined(__ANDROID__) - // Not all filesystems support setting SELinux labels. http://b/23530370. - selinux_android_restorecon(path, 0); + // Not all filesystems support setting SELinux labels. http://b/23530370. + selinux_android_restorecon(path, 0); #endif - // fchown clears the setuid bit - restore it if present. - // Ignore the result of calling fchmod. It's not supported - // by all filesystems, so we don't check for success. b/12441485 - fchmod(fd.get(), mode); - } + // fchown clears the setuid bit - restore it if present. + // Ignore the result of calling fchmod. It's not supported + // by all filesystems, so we don't check for success. b/12441485 + fchmod(fd.get(), mode); + } - { - rc = posix_fadvise(fd.get(), 0, 0, - POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED); + int rc = posix_fadvise(fd.get(), 0, 0, + POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED); if (rc != 0) { D("[ Failed to fadvise: %s ]", strerror(rc)); } - - bool result; - if (compressed) { - result = handle_send_file_compressed(s, std::move(fd), timestamp); - } else { - result = handle_send_file_uncompressed(s, std::move(fd), timestamp, buffer); - } - - if (!result) { - goto fail; - } - - if (!update_capabilities(path, capabilities)) { - SendSyncFailErrno(s, "update_capabilities failed"); - goto fail; - } - - msg.status.id = ID_OKAY; - msg.status.msglen = 0; - return WriteFdExactly(s, &msg.status, sizeof(msg.status)); } + if (!handle_send_file_data(s, std::move(fd), timestamp, compression)) { + goto fail; + } + + if (!update_capabilities(path, capabilities)) { + SendSyncFailErrno(s, "update_capabilities failed"); + goto fail; + } + + msg.status.id = ID_OKAY; + msg.status.msglen = 0; + return WriteFdExactly(s, &msg.status, sizeof(msg.status)); + fail: // If there's a problem on the device, we'll send an ID_FAIL message and // close the socket. Unfortunately the kernel will sometimes throw that @@ -448,7 +437,7 @@ extern bool handle_send_link(int s, const std::string& path, uint32_t* timestamp, std::vector& buffer) __attribute__((error("no symlinks on Windows"))); #else -static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp, +static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp, bool dry_run, std::vector& buffer) { syncmsg msg; @@ -467,19 +456,21 @@ static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp if (!ReadFdExactly(s, &buffer[0], len)) return false; std::string buf_link; - if (!android::base::Readlink(path, &buf_link) || (buf_link != &buffer[0])) { - adb_unlink(path.c_str()); - auto ret = symlink(&buffer[0], path.c_str()); - if (ret && errno == ENOENT) { - if (!secure_mkdirs(Dirname(path))) { - SendSyncFailErrno(s, "secure_mkdirs failed"); + if (!dry_run) { + if (!android::base::Readlink(path, &buf_link) || (buf_link != &buffer[0])) { + adb_unlink(path.c_str()); + auto ret = symlink(&buffer[0], path.c_str()); + if (ret && errno == ENOENT) { + if (!secure_mkdirs(Dirname(path))) { + SendSyncFailErrno(s, "secure_mkdirs failed"); + return false; + } + ret = symlink(&buffer[0], path.c_str()); + } + if (ret) { + SendSyncFailErrno(s, "symlink failed"); return false; } - ret = symlink(&buffer[0], path.c_str()); - } - if (ret) { - SendSyncFailErrno(s, "symlink failed"); - return false; } } @@ -499,12 +490,15 @@ static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp } #endif -static bool send_impl(int s, const std::string& path, mode_t mode, bool compressed, - std::vector& buffer) { +static bool send_impl(int s, const std::string& path, mode_t mode, CompressionType compression, + bool dry_run, std::vector& buffer) { // Don't delete files before copying if they are not "regular" or symlinks. struct stat st; - bool do_unlink = (lstat(path.c_str(), &st) == -1) || S_ISREG(st.st_mode) || - (S_ISLNK(st.st_mode) && !S_ISLNK(mode)); + bool do_unlink = false; + if (!dry_run) { + do_unlink = (lstat(path.c_str(), &st) == -1) || S_ISREG(st.st_mode) || + (S_ISLNK(st.st_mode) && !S_ISLNK(mode)); + } if (do_unlink) { adb_unlink(path.c_str()); } @@ -512,7 +506,7 @@ static bool send_impl(int s, const std::string& path, mode_t mode, bool compress bool result; uint32_t timestamp; if (S_ISLNK(mode)) { - result = handle_send_link(s, path, ×tamp, buffer); + result = handle_send_link(s, path, ×tamp, dry_run, buffer); } else { // Copy user permission bits to "group" and "other" permissions. mode &= 0777; @@ -522,12 +516,12 @@ static bool send_impl(int s, const std::string& path, mode_t mode, bool compress uid_t uid = -1; gid_t gid = -1; uint64_t capabilities = 0; - if (should_use_fs_config(path)) { + if (should_use_fs_config(path) && !dry_run) { adbd_fs_config(path.c_str(), 0, nullptr, &uid, &gid, &mode, &capabilities); } result = handle_send_file(s, path.c_str(), ×tamp, uid, gid, capabilities, mode, - compressed, buffer, do_unlink); + compression, dry_run, buffer, do_unlink); } if (!result) { @@ -560,7 +554,7 @@ static bool do_send_v1(int s, const std::string& spec, std::vector& buffer return false; } - return send_impl(s, path, mode, false, buffer); + return send_impl(s, path, mode, CompressionType::None, false, buffer); } static bool do_send_v2(int s, const std::string& path, std::vector& buffer) { @@ -574,45 +568,80 @@ static bool do_send_v2(int s, const std::string& path, std::vector& buffer PLOG(ERROR) << "failed to read send_v2 setup packet"; } - bool compressed = false; + bool dry_run = false; + std::optional compression; + + uint32_t orig_flags = msg.send_v2_setup.flags; if (msg.send_v2_setup.flags & kSyncFlagBrotli) { msg.send_v2_setup.flags &= ~kSyncFlagBrotli; - compressed = true; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + orig_flags)); + return false; + } + compression = CompressionType::Brotli; } + if (msg.send_v2_setup.flags & kSyncFlagLZ4) { + msg.send_v2_setup.flags &= ~kSyncFlagLZ4; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + orig_flags)); + return false; + } + compression = CompressionType::LZ4; + } + if (msg.send_v2_setup.flags & kSyncFlagDryRun) { + msg.send_v2_setup.flags &= ~kSyncFlagDryRun; + dry_run = true; + } + if (msg.send_v2_setup.flags) { SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.send_v2_setup.flags)); return false; } errno = 0; - return send_impl(s, path, msg.send_v2_setup.mode, compressed, buffer); + return send_impl(s, path, msg.send_v2_setup.mode, compression.value_or(CompressionType::None), + dry_run, buffer); } -static bool recv_uncompressed(borrowed_fd s, unique_fd fd, std::vector& buffer) { - syncmsg msg; - msg.data.id = ID_DATA; - while (true) { - int r = adb_read(fd.get(), &buffer[0], buffer.size() - sizeof(msg.data)); - if (r <= 0) { - if (r == 0) break; - SendSyncFailErrno(s, "read failed"); - return false; - } - msg.data.size = r; +static bool recv_impl(borrowed_fd s, const char* path, CompressionType compression, + std::vector& buffer) { + __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path); - if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) || !WriteFdExactly(s, &buffer[0], r)) { - return false; - } + unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC)); + if (fd < 0) { + SendSyncFailErrno(s, "open failed"); + return false; } - return true; -} + int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE); + if (rc != 0) { + D("[ Failed to fadvise: %s ]", strerror(rc)); + } -static bool recv_compressed(borrowed_fd s, unique_fd fd) { syncmsg msg; msg.data.id = ID_DATA; - BrotliEncoder encoder; + std::variant encoder_storage; + Encoder* encoder; + + switch (compression) { + case CompressionType::None: + encoder = &encoder_storage.emplace(SYNC_DATA_MAX); + break; + + case CompressionType::Brotli: + encoder = &encoder_storage.emplace(SYNC_DATA_MAX); + break; + + case CompressionType::LZ4: + encoder = &encoder_storage.emplace(SYNC_DATA_MAX); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } bool sending = true; while (sending) { @@ -624,15 +653,15 @@ static bool recv_compressed(borrowed_fd s, unique_fd fd) { } if (r == 0) { - encoder.Finish(); + encoder->Finish(); } else { input.resize(r); - encoder.Append(std::move(input)); + encoder->Append(std::move(input)); } while (true) { Block output; - EncodeResult result = encoder.Encode(&output); + EncodeResult result = encoder->Encode(&output); if (result == EncodeResult::Error) { SendSyncFailErrno(s, "compress failed"); return false; @@ -657,42 +686,13 @@ static bool recv_compressed(borrowed_fd s, unique_fd fd) { } } - return true; -} - -static bool recv_impl(borrowed_fd s, const char* path, bool compressed, std::vector& buffer) { - __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path); - - unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC)); - if (fd < 0) { - SendSyncFailErrno(s, "open failed"); - return false; - } - - int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE); - if (rc != 0) { - D("[ Failed to fadvise: %s ]", strerror(rc)); - } - - bool result; - if (compressed) { - result = recv_compressed(s, std::move(fd)); - } else { - result = recv_uncompressed(s, std::move(fd), buffer); - } - - if (!result) { - return false; - } - - syncmsg msg; msg.data.id = ID_DONE; msg.data.size = 0; return WriteFdExactly(s, &msg.data, sizeof(msg.data)); } static bool do_recv_v1(borrowed_fd s, const char* path, std::vector& buffer) { - return recv_impl(s, path, false, buffer); + return recv_impl(s, path, CompressionType::None, buffer); } static bool do_recv_v2(borrowed_fd s, const char* path, std::vector& buffer) { @@ -706,17 +706,33 @@ static bool do_recv_v2(borrowed_fd s, const char* path, std::vector& buffe PLOG(ERROR) << "failed to read recv_v2 setup packet"; } - bool compressed = false; + std::optional compression; + uint32_t orig_flags = msg.recv_v2_setup.flags; if (msg.recv_v2_setup.flags & kSyncFlagBrotli) { msg.recv_v2_setup.flags &= ~kSyncFlagBrotli; - compressed = true; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + orig_flags)); + return false; + } + compression = CompressionType::Brotli; } + if (msg.recv_v2_setup.flags & kSyncFlagLZ4) { + msg.recv_v2_setup.flags &= ~kSyncFlagLZ4; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + orig_flags)); + return false; + } + compression = CompressionType::LZ4; + } + if (msg.recv_v2_setup.flags) { SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.recv_v2_setup.flags)); return false; } - return recv_impl(s, path, compressed, buffer); + return recv_impl(s, path, compression.value_or(CompressionType::None), buffer); } static const char* sync_id_to_name(uint32_t id) { diff --git a/adb/file_sync_protocol.h b/adb/file_sync_protocol.h index fd9a5169e..8f8f85fa5 100644 --- a/adb/file_sync_protocol.h +++ b/adb/file_sync_protocol.h @@ -92,6 +92,15 @@ struct __attribute__((packed)) sync_dent_v2 { enum SyncFlag : uint32_t { kSyncFlagNone = 0, kSyncFlagBrotli = 1, + kSyncFlagLZ4 = 2, + kSyncFlagDryRun = 0x8000'0000U, +}; + +enum class CompressionType { + None, + Any, + Brotli, + LZ4, }; // send_v1 sent the path in a buffer, followed by a comma and the mode as a string. diff --git a/adb/test_device.py b/adb/test_device.py index 6a9ff89ef..f5e4cbb6e 100755 --- a/adb/test_device.py +++ b/adb/test_device.py @@ -77,8 +77,7 @@ def requires_non_root(func): class DeviceTest(unittest.TestCase): - def setUp(self): - self.device = adb.get_device() + device = adb.get_device() class AbbTest(DeviceTest): @@ -753,535 +752,611 @@ def make_random_device_files(device, in_dir, num_files, prefix='device_tmpfile') return files -class FileOperationsTest(DeviceTest): - SCRATCH_DIR = '/data/local/tmp' - DEVICE_TEMP_FILE = SCRATCH_DIR + '/adb_test_file' - DEVICE_TEMP_DIR = SCRATCH_DIR + '/adb_test_dir' +class FileOperationsTest: + class Base(DeviceTest): + SCRATCH_DIR = '/data/local/tmp' + DEVICE_TEMP_FILE = SCRATCH_DIR + '/adb_test_file' + DEVICE_TEMP_DIR = SCRATCH_DIR + '/adb_test_dir' - def _verify_remote(self, checksum, remote_path): - dev_md5, _ = self.device.shell([get_md5_prog(self.device), - remote_path])[0].split() - self.assertEqual(checksum, dev_md5) + def setUp(self): + self.previous_env = os.environ.get("ADB_COMPRESSION") + os.environ["ADB_COMPRESSION"] = self.compression - def _verify_local(self, checksum, local_path): - with open(local_path, 'rb') as host_file: - host_md5 = compute_md5(host_file.read()) - self.assertEqual(host_md5, checksum) + def tearDown(self): + if self.previous_env is None: + del os.environ["ADB_COMPRESSION"] + else: + os.environ["ADB_COMPRESSION"] = self.previous_env - def test_push(self): - """Push a randomly generated file to specified device.""" - kbytes = 512 - tmp = tempfile.NamedTemporaryFile(mode='wb', delete=False) - rand_str = os.urandom(1024 * kbytes) - tmp.write(rand_str) - tmp.close() + def _verify_remote(self, checksum, remote_path): + dev_md5, _ = self.device.shell([get_md5_prog(self.device), + remote_path])[0].split() + self.assertEqual(checksum, dev_md5) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) - self.device.push(local=tmp.name, remote=self.DEVICE_TEMP_FILE) + def _verify_local(self, checksum, local_path): + with open(local_path, 'rb') as host_file: + host_md5 = compute_md5(host_file.read()) + self.assertEqual(host_md5, checksum) - self._verify_remote(compute_md5(rand_str), self.DEVICE_TEMP_FILE) - self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) + def test_push(self): + """Push a randomly generated file to specified device.""" + kbytes = 512 + tmp = tempfile.NamedTemporaryFile(mode='wb', delete=False) + rand_str = os.urandom(1024 * kbytes) + tmp.write(rand_str) + tmp.close() - os.remove(tmp.name) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) + self.device.push(local=tmp.name, remote=self.DEVICE_TEMP_FILE) - def test_push_dir(self): - """Push a randomly generated directory of files to the device.""" - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) + self._verify_remote(compute_md5(rand_str), self.DEVICE_TEMP_FILE) + self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) - try: - host_dir = tempfile.mkdtemp() + os.remove(tmp.name) - # Make sure the temp directory isn't setuid, or else adb will complain. - os.chmod(host_dir, 0o700) - - # Create 32 random files. - temp_files = make_random_host_files(in_dir=host_dir, num_files=32) - self.device.push(host_dir, self.DEVICE_TEMP_DIR) - - for temp_file in temp_files: - remote_path = posixpath.join(self.DEVICE_TEMP_DIR, - os.path.basename(host_dir), - temp_file.base_name) - self._verify_remote(temp_file.checksum, remote_path) + def test_push_dir(self): + """Push a randomly generated directory of files to the device.""" self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - def disabled_test_push_empty(self): - """Push an empty directory to the device.""" - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) + try: + host_dir = tempfile.mkdtemp() - try: - host_dir = tempfile.mkdtemp() + # Make sure the temp directory isn't setuid, or else adb will complain. + os.chmod(host_dir, 0o700) - # Make sure the temp directory isn't setuid, or else adb will complain. - os.chmod(host_dir, 0o700) + # Create 32 random files. + temp_files = make_random_host_files(in_dir=host_dir, num_files=32) + self.device.push(host_dir, self.DEVICE_TEMP_DIR) - # Create an empty directory. - empty_dir_path = os.path.join(host_dir, 'empty') - os.mkdir(empty_dir_path); + for temp_file in temp_files: + remote_path = posixpath.join(self.DEVICE_TEMP_DIR, + os.path.basename(host_dir), + temp_file.base_name) + self._verify_remote(temp_file.checksum, remote_path) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - self.device.push(empty_dir_path, self.DEVICE_TEMP_DIR) - - remote_path = os.path.join(self.DEVICE_TEMP_DIR, "empty") - test_empty_cmd = ["[", "-d", remote_path, "]"] - rc, _, _ = self.device.shell_nocheck(test_empty_cmd) - - self.assertEqual(rc, 0) + def disabled_test_push_empty(self): + """Push an empty directory to the device.""" self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - @unittest.skipIf(sys.platform == "win32", "symlinks require elevated privileges on windows") - def test_push_symlink(self): - """Push a symlink. + try: + host_dir = tempfile.mkdtemp() - Bug: http://b/31491920 - """ - try: - host_dir = tempfile.mkdtemp() + # Make sure the temp directory isn't setuid, or else adb will complain. + os.chmod(host_dir, 0o700) - # Make sure the temp directory isn't setuid, or else adb will - # complain. - os.chmod(host_dir, 0o700) + # Create an empty directory. + empty_dir_path = os.path.join(host_dir, 'empty') + os.mkdir(empty_dir_path); - with open(os.path.join(host_dir, 'foo'), 'w') as f: - f.write('foo') + self.device.push(empty_dir_path, self.DEVICE_TEMP_DIR) - symlink_path = os.path.join(host_dir, 'symlink') - os.symlink('foo', symlink_path) + remote_path = os.path.join(self.DEVICE_TEMP_DIR, "empty") + test_empty_cmd = ["[", "-d", remote_path, "]"] + rc, _, _ = self.device.shell_nocheck(test_empty_cmd) + + self.assertEqual(rc, 0) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + @unittest.skipIf(sys.platform == "win32", "symlinks require elevated privileges on windows") + def test_push_symlink(self): + """Push a symlink. + + Bug: http://b/31491920 + """ + try: + host_dir = tempfile.mkdtemp() + + # Make sure the temp directory isn't setuid, or else adb will + # complain. + os.chmod(host_dir, 0o700) + + with open(os.path.join(host_dir, 'foo'), 'w') as f: + f.write('foo') + + symlink_path = os.path.join(host_dir, 'symlink') + os.symlink('foo', symlink_path) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) + self.device.push(symlink_path, self.DEVICE_TEMP_DIR) + rc, out, _ = self.device.shell_nocheck( + ['cat', posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')]) + self.assertEqual(0, rc) + self.assertEqual(out.strip(), 'foo') + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def test_multiple_push(self): + """Push multiple files to the device in one adb push command. + + Bug: http://b/25324823 + """ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - self.device.push(symlink_path, self.DEVICE_TEMP_DIR) - rc, out, _ = self.device.shell_nocheck( - ['cat', posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')]) - self.assertEqual(0, rc) - self.assertEqual(out.strip(), 'foo') - finally: - if host_dir is not None: - shutil.rmtree(host_dir) - def test_multiple_push(self): - """Push multiple files to the device in one adb push command. - - Bug: http://b/25324823 - """ - - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - - try: - host_dir = tempfile.mkdtemp() - - # Create some random files and a subdirectory containing more files. - temp_files = make_random_host_files(in_dir=host_dir, num_files=4) - - subdir = os.path.join(host_dir, 'subdir') - os.mkdir(subdir) - subdir_temp_files = make_random_host_files(in_dir=subdir, - num_files=4) - - paths = [x.full_path for x in temp_files] - paths.append(subdir) - self.device._simple_call(['push'] + paths + [self.DEVICE_TEMP_DIR]) - - for temp_file in temp_files: - remote_path = posixpath.join(self.DEVICE_TEMP_DIR, - temp_file.base_name) - self._verify_remote(temp_file.checksum, remote_path) - - for subdir_temp_file in subdir_temp_files: - remote_path = posixpath.join(self.DEVICE_TEMP_DIR, - # BROKEN: http://b/25394682 - # 'subdir'; - temp_file.base_name) - self._verify_remote(temp_file.checksum, remote_path) - - - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) - - @requires_non_root - def test_push_error_reporting(self): - """Make sure that errors that occur while pushing a file get reported - - Bug: http://b/26816782 - """ - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(b'\0' * 1024 * 1024) - tmp_file.flush() try: - self.device.push(local=tmp_file.name, remote='/system/') - self.fail('push should not have succeeded') + host_dir = tempfile.mkdtemp() + + # Create some random files and a subdirectory containing more files. + temp_files = make_random_host_files(in_dir=host_dir, num_files=4) + + subdir = os.path.join(host_dir, 'subdir') + os.mkdir(subdir) + subdir_temp_files = make_random_host_files(in_dir=subdir, + num_files=4) + + paths = [x.full_path for x in temp_files] + paths.append(subdir) + self.device._simple_call(['push'] + paths + [self.DEVICE_TEMP_DIR]) + + for temp_file in temp_files: + remote_path = posixpath.join(self.DEVICE_TEMP_DIR, + temp_file.base_name) + self._verify_remote(temp_file.checksum, remote_path) + + for subdir_temp_file in subdir_temp_files: + remote_path = posixpath.join(self.DEVICE_TEMP_DIR, + # BROKEN: http://b/25394682 + # 'subdir'; + temp_file.base_name) + self._verify_remote(temp_file.checksum, remote_path) + + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + @requires_non_root + def test_push_error_reporting(self): + """Make sure that errors that occur while pushing a file get reported + + Bug: http://b/26816782 + """ + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(b'\0' * 1024 * 1024) + tmp_file.flush() + try: + self.device.push(local=tmp_file.name, remote='/system/') + self.fail('push should not have succeeded') + except subprocess.CalledProcessError as e: + output = e.output + + self.assertTrue(b'Permission denied' in output or + b'Read-only file system' in output) + + @requires_non_root + def test_push_directory_creation(self): + """Regression test for directory creation. + + Bug: http://b/110953234 + """ + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(b'\0' * 1024 * 1024) + tmp_file.flush() + remote_path = self.DEVICE_TEMP_DIR + '/test_push_directory_creation' + self.device.shell(['rm', '-rf', remote_path]) + + remote_path += '/filename' + self.device.push(local=tmp_file.name, remote=remote_path) + + def disabled_test_push_multiple_slash_root(self): + """Regression test for pushing to //data/local/tmp. + + Bug: http://b/141311284 + + Disabled because this broken on the adbd side as well: b/141943968 + """ + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write('\0' * 1024 * 1024) + tmp_file.flush() + remote_path = '/' + self.DEVICE_TEMP_DIR + '/test_push_multiple_slash_root' + self.device.shell(['rm', '-rf', remote_path]) + self.device.push(local=tmp_file.name, remote=remote_path) + + def _test_pull(self, remote_file, checksum): + tmp_write = tempfile.NamedTemporaryFile(mode='wb', delete=False) + tmp_write.close() + self.device.pull(remote=remote_file, local=tmp_write.name) + with open(tmp_write.name, 'rb') as tmp_read: + host_contents = tmp_read.read() + host_md5 = compute_md5(host_contents) + self.assertEqual(checksum, host_md5) + os.remove(tmp_write.name) + + @requires_non_root + def test_pull_error_reporting(self): + self.device.shell(['touch', self.DEVICE_TEMP_FILE]) + self.device.shell(['chmod', 'a-rwx', self.DEVICE_TEMP_FILE]) + + try: + output = self.device.pull(remote=self.DEVICE_TEMP_FILE, local='x') except subprocess.CalledProcessError as e: output = e.output - self.assertTrue(b'Permission denied' in output or - b'Read-only file system' in output) + self.assertIn(b'Permission denied', output) - @requires_non_root - def test_push_directory_creation(self): - """Regression test for directory creation. + self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) - Bug: http://b/110953234 - """ - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(b'\0' * 1024 * 1024) - tmp_file.flush() - remote_path = self.DEVICE_TEMP_DIR + '/test_push_directory_creation' - self.device.shell(['rm', '-rf', remote_path]) + def test_pull(self): + """Pull a randomly generated file from specified device.""" + kbytes = 512 + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) + cmd = ['dd', 'if=/dev/urandom', + 'of={}'.format(self.DEVICE_TEMP_FILE), 'bs=1024', + 'count={}'.format(kbytes)] + self.device.shell(cmd) + dev_md5, _ = self.device.shell( + [get_md5_prog(self.device), self.DEVICE_TEMP_FILE])[0].split() + self._test_pull(self.DEVICE_TEMP_FILE, dev_md5) + self.device.shell_nocheck(['rm', self.DEVICE_TEMP_FILE]) - remote_path += '/filename' - self.device.push(local=tmp_file.name, remote=remote_path) + def test_pull_dir(self): + """Pull a randomly generated directory of files from the device.""" + try: + host_dir = tempfile.mkdtemp() - def disabled_test_push_multiple_slash_root(self): - """Regression test for pushing to //data/local/tmp. + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) - Bug: http://b/141311284 + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) - Disabled because this broken on the adbd side as well: b/141943968 - """ - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write('\0' * 1024 * 1024) - tmp_file.flush() - remote_path = '/' + self.DEVICE_TEMP_DIR + '/test_push_multiple_slash_root' - self.device.shell(['rm', '-rf', remote_path]) - self.device.push(local=tmp_file.name, remote=remote_path) + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) - def _test_pull(self, remote_file, checksum): - tmp_write = tempfile.NamedTemporaryFile(mode='wb', delete=False) - tmp_write.close() - self.device.pull(remote=remote_file, local=tmp_write.name) - with open(tmp_write.name, 'rb') as tmp_read: - host_contents = tmp_read.read() - host_md5 = compute_md5(host_contents) - self.assertEqual(checksum, host_md5) - os.remove(tmp_write.name) + for temp_file in temp_files: + host_path = os.path.join( + host_dir, posixpath.basename(self.DEVICE_TEMP_DIR), + temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) - @requires_non_root - def test_pull_error_reporting(self): - self.device.shell(['touch', self.DEVICE_TEMP_FILE]) - self.device.shell(['chmod', 'a-rwx', self.DEVICE_TEMP_FILE]) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - try: - output = self.device.pull(remote=self.DEVICE_TEMP_FILE, local='x') - except subprocess.CalledProcessError as e: - output = e.output + def test_pull_dir_symlink(self): + """Pull a directory into a symlink to a directory. - self.assertIn(b'Permission denied', output) + Bug: http://b/27362811 + """ + if os.name != 'posix': + raise unittest.SkipTest('requires POSIX') - self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) + try: + host_dir = tempfile.mkdtemp() + real_dir = os.path.join(host_dir, 'dir') + symlink = os.path.join(host_dir, 'symlink') + os.mkdir(real_dir) + os.symlink(real_dir, symlink) - def test_pull(self): - """Pull a randomly generated file from specified device.""" - kbytes = 512 - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) - cmd = ['dd', 'if=/dev/urandom', - 'of={}'.format(self.DEVICE_TEMP_FILE), 'bs=1024', - 'count={}'.format(kbytes)] - self.device.shell(cmd) - dev_md5, _ = self.device.shell( - [get_md5_prog(self.device), self.DEVICE_TEMP_FILE])[0].split() - self._test_pull(self.DEVICE_TEMP_FILE, dev_md5) - self.device.shell_nocheck(['rm', self.DEVICE_TEMP_FILE]) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) - def test_pull_dir(self): - """Pull a randomly generated directory of files from the device.""" - try: - host_dir = tempfile.mkdtemp() + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=symlink) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + for temp_file in temp_files: + host_path = os.path.join( + real_dir, posixpath.basename(self.DEVICE_TEMP_DIR), + temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + def test_pull_dir_symlink_collision(self): + """Pull a directory into a colliding symlink to directory.""" + if os.name != 'posix': + raise unittest.SkipTest('requires POSIX') + + try: + host_dir = tempfile.mkdtemp() + real_dir = os.path.join(host_dir, 'real') + tmp_dirname = os.path.basename(self.DEVICE_TEMP_DIR) + symlink = os.path.join(host_dir, tmp_dirname) + os.mkdir(real_dir) + os.symlink(real_dir, symlink) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) + + for temp_file in temp_files: + host_path = os.path.join(real_dir, temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def test_pull_dir_nonexistent(self): + """Pull a directory of files from the device to a nonexistent path.""" + try: + host_dir = tempfile.mkdtemp() + dest_dir = os.path.join(host_dir, 'dest') + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=dest_dir) + + for temp_file in temp_files: + host_path = os.path.join(dest_dir, temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + # selinux prevents adbd from accessing symlinks on /data/local/tmp. + def disabled_test_pull_symlink_dir(self): + """Pull a symlink to a directory of symlinks to files.""" + try: + host_dir = tempfile.mkdtemp() + + remote_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'contents') + remote_links = posixpath.join(self.DEVICE_TEMP_DIR, 'links') + remote_symlink = posixpath.join(self.DEVICE_TEMP_DIR, 'symlink') + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', remote_dir, remote_links]) + self.device.shell(['ln', '-s', remote_links, remote_symlink]) + + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=remote_dir, num_files=32) + + for temp_file in temp_files: + self.device.shell( + ['ln', '-s', '../contents/{}'.format(temp_file.base_name), + posixpath.join(remote_links, temp_file.base_name)]) + + self.device.pull(remote=remote_symlink, local=host_dir) + + for temp_file in temp_files: + host_path = os.path.join( + host_dir, 'symlink', temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def test_pull_empty(self): + """Pull a directory containing an empty directory from the device.""" + try: + host_dir = tempfile.mkdtemp() + + remote_empty_path = posixpath.join(self.DEVICE_TEMP_DIR, 'empty') + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', remote_empty_path]) + + self.device.pull(remote=remote_empty_path, local=host_dir) + self.assertTrue(os.path.isdir(os.path.join(host_dir, 'empty'))) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def test_multiple_pull(self): + """Pull a randomly generated directory of files from the device.""" + + try: + host_dir = tempfile.mkdtemp() + + subdir = posixpath.join(self.DEVICE_TEMP_DIR, 'subdir') + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', subdir]) + + # Create some random files and a subdirectory containing more files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=4) + + subdir_temp_files = make_random_device_files( + self.device, in_dir=subdir, num_files=4, prefix='subdir_') + + paths = [x.full_path for x in temp_files] + paths.append(subdir) + self.device._simple_call(['pull'] + paths + [host_dir]) + + for temp_file in temp_files: + local_path = os.path.join(host_dir, temp_file.base_name) + self._verify_local(temp_file.checksum, local_path) + + for subdir_temp_file in subdir_temp_files: + local_path = os.path.join(host_dir, + 'subdir', + subdir_temp_file.base_name) + self._verify_local(subdir_temp_file.checksum, local_path) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def verify_sync(self, device, temp_files, device_dir): + """Verifies that a list of temp files was synced to the device.""" + # Confirm that every file on the device mirrors that on the host. for temp_file in temp_files: - host_path = os.path.join( - host_dir, posixpath.basename(self.DEVICE_TEMP_DIR), - temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + device_full_path = posixpath.join( + device_dir, temp_file.base_name) + dev_md5, _ = device.shell( + [get_md5_prog(self.device), device_full_path])[0].split() + self.assertEqual(temp_file.checksum, dev_md5) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + def test_sync(self): + """Sync a host directory to the data partition.""" - def test_pull_dir_symlink(self): - """Pull a directory into a symlink to a directory. + try: + base_dir = tempfile.mkdtemp() - Bug: http://b/27362811 - """ - if os.name != 'posix': - raise unittest.SkipTest('requires POSIX') + # Create mirror device directory hierarchy within base_dir. + full_dir_path = base_dir + self.DEVICE_TEMP_DIR + os.makedirs(full_dir_path) - try: - host_dir = tempfile.mkdtemp() - real_dir = os.path.join(host_dir, 'dir') - symlink = os.path.join(host_dir, 'symlink') - os.mkdir(real_dir) - os.symlink(real_dir, symlink) + # Create 32 random files within the host mirror. + temp_files = make_random_host_files( + in_dir=full_dir_path, num_files=32) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + # Clean up any stale files on the device. + device = adb.get_device() # pylint: disable=no-member + device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + old_product_out = os.environ.get('ANDROID_PRODUCT_OUT') + os.environ['ANDROID_PRODUCT_OUT'] = base_dir + device.sync('data') + if old_product_out is None: + del os.environ['ANDROID_PRODUCT_OUT'] + else: + os.environ['ANDROID_PRODUCT_OUT'] = old_product_out - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=symlink) + self.verify_sync(device, temp_files, self.DEVICE_TEMP_DIR) - for temp_file in temp_files: - host_path = os.path.join( - real_dir, posixpath.basename(self.DEVICE_TEMP_DIR), - temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + #self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if base_dir is not None: + shutil.rmtree(base_dir) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + def test_push_sync(self): + """Sync a host directory to a specific path.""" - def test_pull_dir_symlink_collision(self): - """Pull a directory into a colliding symlink to directory.""" - if os.name != 'posix': - raise unittest.SkipTest('requires POSIX') + try: + temp_dir = tempfile.mkdtemp() + temp_files = make_random_host_files(in_dir=temp_dir, num_files=32) - try: - host_dir = tempfile.mkdtemp() - real_dir = os.path.join(host_dir, 'real') - tmp_dirname = os.path.basename(self.DEVICE_TEMP_DIR) - symlink = os.path.join(host_dir, tmp_dirname) - os.mkdir(real_dir) - os.symlink(real_dir, symlink) + device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'sync_src_dst') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + # Clean up any stale files on the device. + device = adb.get_device() # pylint: disable=no-member + device.shell(['rm', '-rf', device_dir]) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + device.push(temp_dir, device_dir, sync=True) - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) + self.verify_sync(device, temp_files, device_dir) - for temp_file in temp_files: - host_path = os.path.join(real_dir, temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + def test_push_dry_run_nonexistent_file(self): + """Push with dry run.""" - def test_pull_dir_nonexistent(self): - """Pull a directory of files from the device to a nonexistent path.""" - try: - host_dir = tempfile.mkdtemp() - dest_dir = os.path.join(host_dir, 'dest') + for file_size in [8, 1024 * 1024]: + try: + device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'push_dry_run') + device_file = posixpath.join(device_dir, 'file') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', device_dir]) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + host_dir = tempfile.mkdtemp() + host_file = posixpath.join(host_dir, 'file') - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=dest_dir) + with open(host_file, "w") as f: + f.write('x' * file_size) - for temp_file in temp_files: - host_path = os.path.join(dest_dir, temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + self.device._simple_call(['push', '-n', host_file, device_file]) + rc, _, _ = self.device.shell_nocheck(['[', '-e', device_file, ']']) + self.assertNotEqual(0, rc) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - # selinux prevents adbd from accessing symlinks on /data/local/tmp. - def disabled_test_pull_symlink_dir(self): - """Pull a symlink to a directory of symlinks to files.""" - try: - host_dir = tempfile.mkdtemp() + def test_push_dry_run_existent_file(self): + """Push with dry run.""" - remote_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'contents') - remote_links = posixpath.join(self.DEVICE_TEMP_DIR, 'links') - remote_symlink = posixpath.join(self.DEVICE_TEMP_DIR, 'symlink') + for file_size in [8, 1024 * 1024]: + try: + device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'push_dry_run') + device_file = posixpath.join(device_dir, 'file') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', remote_dir, remote_links]) - self.device.shell(['ln', '-s', remote_links, remote_symlink]) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', device_dir]) + self.device.shell(['echo', 'foo', '>', device_file]) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=remote_dir, num_files=32) + host_dir = tempfile.mkdtemp() + host_file = posixpath.join(host_dir, 'file') - for temp_file in temp_files: - self.device.shell( - ['ln', '-s', '../contents/{}'.format(temp_file.base_name), - posixpath.join(remote_links, temp_file.base_name)]) + with open(host_file, "w") as f: + f.write('x' * file_size) - self.device.pull(remote=remote_symlink, local=host_dir) + self.device._simple_call(['push', '-n', host_file, device_file]) + stdout, stderr = self.device.shell(['cat', device_file]) + self.assertEqual(stdout.strip(), "foo") - for temp_file in temp_files: - host_path = os.path.join( - host_dir, 'symlink', temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + def test_unicode_paths(self): + """Ensure that we can support non-ASCII paths, even on Windows.""" + name = u'로보카 폴리' - def test_pull_empty(self): - """Pull a directory containing an empty directory from the device.""" - try: - host_dir = tempfile.mkdtemp() + self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) + remote_path = u'/data/local/tmp/adb-test-{}'.format(name) - remote_empty_path = posixpath.join(self.DEVICE_TEMP_DIR, 'empty') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', remote_empty_path]) + ## push. + tf = tempfile.NamedTemporaryFile('wb', suffix=name, delete=False) + tf.close() + self.device.push(tf.name, remote_path) + os.remove(tf.name) + self.assertFalse(os.path.exists(tf.name)) - self.device.pull(remote=remote_empty_path, local=host_dir) - self.assertTrue(os.path.isdir(os.path.join(host_dir, 'empty'))) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + # Verify that the device ended up with the expected UTF-8 path + output = self.device.shell( + ['ls', '/data/local/tmp/adb-test-*'])[0].strip() + self.assertEqual(remote_path, output) - def test_multiple_pull(self): - """Pull a randomly generated directory of files from the device.""" + # pull. + self.device.pull(remote_path, tf.name) + self.assertTrue(os.path.exists(tf.name)) + os.remove(tf.name) + self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) - try: - host_dir = tempfile.mkdtemp() - subdir = posixpath.join(self.DEVICE_TEMP_DIR, 'subdir') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', subdir]) +class FileOperationsTestUncompressed(FileOperationsTest.Base): + compression = "none" - # Create some random files and a subdirectory containing more files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=4) - subdir_temp_files = make_random_device_files( - self.device, in_dir=subdir, num_files=4, prefix='subdir_') +class FileOperationsTestBrotli(FileOperationsTest.Base): + compression = "brotli" - paths = [x.full_path for x in temp_files] - paths.append(subdir) - self.device._simple_call(['pull'] + paths + [host_dir]) - for temp_file in temp_files: - local_path = os.path.join(host_dir, temp_file.base_name) - self._verify_local(temp_file.checksum, local_path) - - for subdir_temp_file in subdir_temp_files: - local_path = os.path.join(host_dir, - 'subdir', - subdir_temp_file.base_name) - self._verify_local(subdir_temp_file.checksum, local_path) - - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) - - def verify_sync(self, device, temp_files, device_dir): - """Verifies that a list of temp files was synced to the device.""" - # Confirm that every file on the device mirrors that on the host. - for temp_file in temp_files: - device_full_path = posixpath.join( - device_dir, temp_file.base_name) - dev_md5, _ = device.shell( - [get_md5_prog(self.device), device_full_path])[0].split() - self.assertEqual(temp_file.checksum, dev_md5) - - def test_sync(self): - """Sync a host directory to the data partition.""" - - try: - base_dir = tempfile.mkdtemp() - - # Create mirror device directory hierarchy within base_dir. - full_dir_path = base_dir + self.DEVICE_TEMP_DIR - os.makedirs(full_dir_path) - - # Create 32 random files within the host mirror. - temp_files = make_random_host_files( - in_dir=full_dir_path, num_files=32) - - # Clean up any stale files on the device. - device = adb.get_device() # pylint: disable=no-member - device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - - old_product_out = os.environ.get('ANDROID_PRODUCT_OUT') - os.environ['ANDROID_PRODUCT_OUT'] = base_dir - device.sync('data') - if old_product_out is None: - del os.environ['ANDROID_PRODUCT_OUT'] - else: - os.environ['ANDROID_PRODUCT_OUT'] = old_product_out - - self.verify_sync(device, temp_files, self.DEVICE_TEMP_DIR) - - #self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if base_dir is not None: - shutil.rmtree(base_dir) - - def test_push_sync(self): - """Sync a host directory to a specific path.""" - - try: - temp_dir = tempfile.mkdtemp() - temp_files = make_random_host_files(in_dir=temp_dir, num_files=32) - - device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'sync_src_dst') - - # Clean up any stale files on the device. - device = adb.get_device() # pylint: disable=no-member - device.shell(['rm', '-rf', device_dir]) - - device.push(temp_dir, device_dir, sync=True) - - self.verify_sync(device, temp_files, device_dir) - - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if temp_dir is not None: - shutil.rmtree(temp_dir) - - def test_unicode_paths(self): - """Ensure that we can support non-ASCII paths, even on Windows.""" - name = u'로보카 폴리' - - self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) - remote_path = u'/data/local/tmp/adb-test-{}'.format(name) - - ## push. - tf = tempfile.NamedTemporaryFile('wb', suffix=name, delete=False) - tf.close() - self.device.push(tf.name, remote_path) - os.remove(tf.name) - self.assertFalse(os.path.exists(tf.name)) - - # Verify that the device ended up with the expected UTF-8 path - output = self.device.shell( - ['ls', '/data/local/tmp/adb-test-*'])[0].strip() - self.assertEqual(remote_path, output) - - # pull. - self.device.pull(remote_path, tf.name) - self.assertTrue(os.path.exists(tf.name)) - os.remove(tf.name) - self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) +class FileOperationsTestLZ4(FileOperationsTest.Base): + compression = "lz4" class DeviceOfflineTest(DeviceTest): diff --git a/adb/transport.cpp b/adb/transport.cpp index e06dbe327..cc2e0e3b1 100644 --- a/adb/transport.cpp +++ b/adb/transport.cpp @@ -84,6 +84,8 @@ const char* const kFeatureRemountShell = "remount_shell"; const char* const kFeatureTrackApp = "track_app"; const char* const kFeatureSendRecv2 = "sendrecv_v2"; const char* const kFeatureSendRecv2Brotli = "sendrecv_v2_brotli"; +const char* const kFeatureSendRecv2LZ4 = "sendrecv_v2_lz4"; +const char* const kFeatureSendRecv2DryRunSend = "sendrecv_v2_dry_run_send"; namespace { @@ -1183,6 +1185,8 @@ const FeatureSet& supported_features() { kFeatureTrackApp, kFeatureSendRecv2, kFeatureSendRecv2Brotli, + kFeatureSendRecv2LZ4, + kFeatureSendRecv2DryRunSend, // Increment ADB_SERVER_VERSION when adding a feature that adbd needs // to know about. Otherwise, the client can be stuck running an old // version of the server even after upgrading their copy of adb. diff --git a/adb/transport.h b/adb/transport.h index b1984db30..4b2e00043 100644 --- a/adb/transport.h +++ b/adb/transport.h @@ -88,6 +88,10 @@ extern const char* const kFeatureTrackApp; extern const char* const kFeatureSendRecv2; // adbd supports brotli for send/recv v2. extern const char* const kFeatureSendRecv2Brotli; +// adbd supports LZ4 for send/recv v2. +extern const char* const kFeatureSendRecv2LZ4; +// adbd supports dry-run send for send/recv v2. +extern const char* const kFeatureSendRecv2DryRunSend; TransportId NextTransportId(); diff --git a/adb/types.cpp b/adb/types.cpp index 26b77ab62..9cdf32b93 100644 --- a/adb/types.cpp +++ b/adb/types.cpp @@ -51,7 +51,7 @@ void IOVector::drop_front(IOVector::size_type len) { auto dropped = 0u; while (dropped < len) { const auto next = chain_[start_index_].size() - begin_offset_; - if (dropped + next < len) { + if (dropped + next <= len) { pop_front_block(); dropped += next; } else { diff --git a/adb/types.h b/adb/types.h index deca7eafb..620aa8eb7 100644 --- a/adb/types.h +++ b/adb/types.h @@ -155,7 +155,7 @@ struct IOVector { return nullptr; } - return chain_.front().data() + begin_offset_; + return chain_[start_index_].data() + begin_offset_; } size_type front_size() const { @@ -163,7 +163,7 @@ struct IOVector { return 0; } - return chain_.front().size() - begin_offset_; + return chain_[start_index_].size() - begin_offset_; } size_type size() const { return chain_length_ - begin_offset_; } diff --git a/adb/types_test.cpp b/adb/types_test.cpp index 2c99f9512..41fa1db25 100644 --- a/adb/types_test.cpp +++ b/adb/types_test.cpp @@ -117,3 +117,20 @@ TEST(IOVector, misaligned_split) { ASSERT_EQ(1ULL, bc.size()); ASSERT_EQ(create_block("x"), bc.coalesce()); } + +TEST(IOVector, drop_front) { + IOVector vec; + + vec.append(create_block('x', 2)); + vec.append(create_block('y', 1000)); + ASSERT_EQ(2U, vec.front_size()); + ASSERT_EQ(1002U, vec.size()); + + vec.drop_front(1); + ASSERT_EQ(1U, vec.front_size()); + ASSERT_EQ(1001U, vec.size()); + + vec.drop_front(1); + ASSERT_EQ(1000U, vec.front_size()); + ASSERT_EQ(1000U, vec.size()); +}