diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index f8e4b7aec..db50e5824 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -403,3 +403,30 @@ cc_test { auto_gen_config: true, require_root: false, } + +cc_binary { + name: "make_cow_from_ab_ota", + host_supported: true, + device_supported: false, + static_libs: [ + "libbase", + "libbspatch", + "libbrotli", + "libbz", + "libchrome", + "libcrypto", + "libgflags", + "liblog", + "libprotobuf-cpp-lite", + "libpuffpatch", + "libsnapshot_cow", + "libsparse", + "libxz", + "libz", + "libziparchive", + "update_metadata-protos", + ], + srcs: [ + "make_cow_from_ab_ota.cpp", + ], +} diff --git a/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp b/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp new file mode 100644 index 000000000..0b40fd60b --- /dev/null +++ b/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp @@ -0,0 +1,690 @@ +// +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace android { +namespace snapshot { + +using android::base::borrowed_fd; +using android::base::unique_fd; +using chromeos_update_engine::DeltaArchiveManifest; +using chromeos_update_engine::Extent; +using chromeos_update_engine::InstallOperation; +using chromeos_update_engine::PartitionUpdate; + +static constexpr uint64_t kBlockSize = 4096; + +DEFINE_string(source_tf, "", "Source target files (dir or zip file) for incremental payloads"); +DEFINE_string(compression, "gz", "Compression type to use (none or gz)"); + +void MyLogger(android::base::LogId, android::base::LogSeverity severity, const char*, const char*, + unsigned int, const char* message) { + if (severity == android::base::ERROR) { + fprintf(stderr, "%s\n", message); + } else { + fprintf(stdout, "%s\n", message); + } +} + +uint64_t ToLittleEndian(uint64_t value) { + union { + uint64_t u64; + char bytes[8]; + } packed; + packed.u64 = value; + std::swap(packed.bytes[0], packed.bytes[7]); + std::swap(packed.bytes[1], packed.bytes[6]); + std::swap(packed.bytes[2], packed.bytes[5]); + std::swap(packed.bytes[3], packed.bytes[4]); + return packed.u64; +} + +class PayloadConverter final { + public: + PayloadConverter(const std::string& in_file, const std::string& out_dir) + : in_file_(in_file), out_dir_(out_dir), source_tf_zip_(nullptr, &CloseArchive) {} + + bool Run(); + + private: + bool OpenPayload(); + bool OpenSourceTargetFiles(); + bool ProcessPartition(const PartitionUpdate& update); + bool ProcessOperation(const InstallOperation& op); + bool ProcessZero(const InstallOperation& op); + bool ProcessCopy(const InstallOperation& op); + bool ProcessReplace(const InstallOperation& op); + bool ProcessDiff(const InstallOperation& op); + borrowed_fd OpenSourceImage(); + + std::string in_file_; + std::string out_dir_; + unique_fd in_fd_; + uint64_t payload_offset_ = 0; + DeltaArchiveManifest manifest_; + std::unordered_set dap_; + unique_fd source_tf_fd_; + std::unique_ptr source_tf_zip_; + + // Updated during ProcessPartition(). + std::string partition_name_; + std::unique_ptr writer_; + unique_fd source_image_; +}; + +bool PayloadConverter::Run() { + if (!OpenPayload()) { + return false; + } + + if (manifest_.has_dynamic_partition_metadata()) { + const auto& dpm = manifest_.dynamic_partition_metadata(); + for (const auto& group : dpm.groups()) { + for (const auto& partition : group.partition_names()) { + dap_.emplace(partition); + } + } + } + + if (dap_.empty()) { + LOG(ERROR) << "No dynamic partitions found."; + return false; + } + + if (!OpenSourceTargetFiles()) { + return false; + } + + for (const auto& update : manifest_.partitions()) { + if (!ProcessPartition(update)) { + return false; + } + writer_ = nullptr; + source_image_.reset(); + } + return true; +} + +bool PayloadConverter::OpenSourceTargetFiles() { + if (FLAGS_source_tf.empty()) { + return true; + } + + source_tf_fd_.reset(open(FLAGS_source_tf.c_str(), O_RDONLY)); + if (source_tf_fd_ < 0) { + LOG(ERROR) << "open failed: " << FLAGS_source_tf; + return false; + } + + struct stat s; + if (fstat(source_tf_fd_.get(), &s) < 0) { + LOG(ERROR) << "fstat failed: " << FLAGS_source_tf; + return false; + } + if (S_ISDIR(s.st_mode)) { + return true; + } + + // Otherwise, assume it's a zip file. + ZipArchiveHandle handle; + if (OpenArchiveFd(source_tf_fd_.get(), FLAGS_source_tf.c_str(), &handle, false)) { + LOG(ERROR) << "Could not open " << FLAGS_source_tf << " as a zip archive."; + return false; + } + source_tf_zip_.reset(handle); + return true; +} + +bool PayloadConverter::ProcessPartition(const PartitionUpdate& update) { + auto partition_name = update.partition_name(); + if (dap_.find(partition_name) == dap_.end()) { + // Skip non-DAP partitions. + return true; + } + + auto path = out_dir_ + "/" + partition_name + ".cow"; + unique_fd fd(open(path.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, 0644)); + if (fd < 0) { + PLOG(ERROR) << "open failed: " << path; + return false; + } + + CowOptions options; + options.block_size = kBlockSize; + options.compression = FLAGS_compression; + + writer_ = std::make_unique(options); + if (!writer_->Initialize(std::move(fd))) { + LOG(ERROR) << "Unable to initialize COW writer"; + return false; + } + + partition_name_ = partition_name; + + for (const auto& op : update.operations()) { + if (!ProcessOperation(op)) { + return false; + } + } + + if (!writer_->Finalize()) { + LOG(ERROR) << "Unable to finalize COW for " << partition_name; + return false; + } + return true; +} + +bool PayloadConverter::ProcessOperation(const InstallOperation& op) { + switch (op.type()) { + case InstallOperation::SOURCE_COPY: + return ProcessCopy(op); + case InstallOperation::BROTLI_BSDIFF: + case InstallOperation::PUFFDIFF: + return ProcessDiff(op); + case InstallOperation::REPLACE: + case InstallOperation::REPLACE_XZ: + case InstallOperation::REPLACE_BZ: + return ProcessReplace(op); + case InstallOperation::ZERO: + return ProcessZero(op); + default: + LOG(ERROR) << "Unsupported op: " << (int)op.type(); + return false; + } + return true; +} + +bool PayloadConverter::ProcessZero(const InstallOperation& op) { + for (const auto& extent : op.dst_extents()) { + if (!writer_->AddZeroBlocks(extent.start_block(), extent.num_blocks())) { + LOG(ERROR) << "Could not add zero operation"; + return false; + } + } + return true; +} + +template +static uint64_t SizeOfAllExtents(const T& extents) { + uint64_t total = 0; + for (const auto& extent : extents) { + total += extent.num_blocks() * kBlockSize; + } + return total; +} + +class PuffInputStream final : public puffin::StreamInterface { + public: + PuffInputStream(uint8_t* buffer, size_t length) : buffer_(buffer), length_(length) {} + + bool GetSize(uint64_t* size) const override { + *size = length_; + return true; + } + bool GetOffset(uint64_t* offset) const override { + *offset = pos_; + return true; + } + bool Seek(uint64_t offset) override { + if (offset > length_) return false; + pos_ = offset; + return true; + } + bool Read(void* buffer, size_t length) override { + if (length_ - pos_ < length) return false; + memcpy(buffer, buffer_ + pos_, length); + pos_ += length; + return true; + } + bool Write(const void*, size_t) override { return false; } + bool Close() override { return true; } + + private: + uint8_t* buffer_; + size_t length_; + size_t pos_; +}; + +class PuffOutputStream final : public puffin::StreamInterface { + public: + PuffOutputStream(std::vector& stream) : stream_(stream), pos_(0) {} + + bool GetSize(uint64_t* size) const override { + *size = stream_.size(); + return true; + } + bool GetOffset(uint64_t* offset) const override { + *offset = pos_; + return true; + } + bool Seek(uint64_t offset) override { + if (offset > stream_.size()) { + return false; + } + pos_ = offset; + return true; + } + bool Read(void* buffer, size_t length) override { + if (stream_.size() - pos_ < length) { + return false; + } + memcpy(buffer, &stream_[0] + pos_, length); + pos_ += length; + return true; + } + bool Write(const void* buffer, size_t length) override { + auto remaining = stream_.size() - pos_; + if (remaining < length) { + stream_.resize(stream_.size() + (length - remaining)); + } + memcpy(&stream_[0] + pos_, buffer, length); + pos_ += length; + return true; + } + bool Close() override { return true; } + + private: + std::vector& stream_; + size_t pos_; +}; + +bool PayloadConverter::ProcessDiff(const InstallOperation& op) { + auto source_image = OpenSourceImage(); + if (source_image < 0) { + return false; + } + + uint64_t src_length = SizeOfAllExtents(op.src_extents()); + auto src = std::make_unique(src_length); + size_t src_pos = 0; + + // Read source bytes. + for (const auto& extent : op.src_extents()) { + uint64_t offset = extent.start_block() * kBlockSize; + if (lseek(source_image.get(), offset, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek source image failed"; + return false; + } + + uint64_t size = extent.num_blocks() * kBlockSize; + CHECK(src_length - src_pos >= size); + if (!android::base::ReadFully(source_image, src.get() + src_pos, size)) { + PLOG(ERROR) << "read source image failed"; + return false; + } + src_pos += size; + } + CHECK(src_pos == src_length); + + // Read patch bytes. + auto patch = std::make_unique(op.data_length()); + if (lseek(in_fd_.get(), payload_offset_ + op.data_offset(), SEEK_SET) < 0) { + PLOG(ERROR) << "lseek payload failed"; + return false; + } + if (!android::base::ReadFully(in_fd_, patch.get(), op.data_length())) { + PLOG(ERROR) << "read payload failed"; + return false; + } + + std::vector dest(SizeOfAllExtents(op.dst_extents())); + + // Apply the diff. + if (op.type() == InstallOperation::BROTLI_BSDIFF) { + size_t dest_pos = 0; + auto sink = [&](const uint8_t* data, size_t length) -> size_t { + CHECK(dest.size() - dest_pos >= length); + memcpy(&dest[dest_pos], data, length); + dest_pos += length; + return length; + }; + if (int rv = bsdiff::bspatch(src.get(), src_pos, patch.get(), op.data_length(), sink)) { + LOG(ERROR) << "bspatch failed, error code " << rv; + return false; + } + } else if (op.type() == InstallOperation::PUFFDIFF) { + auto src_stream = std::make_unique(src.get(), src_length); + auto dest_stream = std::make_unique(dest); + bool ok = PuffPatch(std::move(src_stream), std::move(dest_stream), patch.get(), + op.data_length()); + if (!ok) { + LOG(ERROR) << "puffdiff operation failed to apply"; + return false; + } + } else { + LOG(ERROR) << "unsupported diff operation: " << op.type(); + return false; + } + + // Write the final blocks to the COW. + size_t dest_pos = 0; + for (const auto& extent : op.dst_extents()) { + uint64_t size = extent.num_blocks() * kBlockSize; + CHECK(dest.size() - dest_pos >= size); + + if (!writer_->AddRawBlocks(extent.start_block(), &dest[dest_pos], size)) { + return false; + } + dest_pos += size; + } + return true; +} + +borrowed_fd PayloadConverter::OpenSourceImage() { + if (source_image_ >= 0) { + return source_image_; + } + + unique_fd unzip_fd; + + auto local_path = "IMAGES/" + partition_name_ + ".img"; + if (source_tf_zip_) { + { + TemporaryFile tmp; + if (tmp.fd < 0) { + PLOG(ERROR) << "mkstemp failed"; + return -1; + } + unzip_fd.reset(tmp.release()); + } + + ZipEntry64 entry; + if (FindEntry(source_tf_zip_.get(), local_path, &entry)) { + LOG(ERROR) << "not found in archive: " << local_path; + return -1; + } + if (ExtractEntryToFile(source_tf_zip_.get(), &entry, unzip_fd.get())) { + LOG(ERROR) << "could not extract " << local_path; + return -1; + } + if (lseek(unzip_fd.get(), 0, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek failed"; + return -1; + } + } else if (source_tf_fd_ >= 0) { + unzip_fd.reset(openat(source_tf_fd_.get(), local_path.c_str(), O_RDONLY)); + if (unzip_fd < 0) { + PLOG(ERROR) << "open failed: " << FLAGS_source_tf << "/" << local_path; + return -1; + } + } else { + LOG(ERROR) << "No source target files package was specified; need -source_tf"; + return -1; + } + + std::unique_ptr s( + sparse_file_import(unzip_fd.get(), false, false), &sparse_file_destroy); + if (s) { + TemporaryFile tmp; + if (tmp.fd < 0) { + PLOG(ERROR) << "mkstemp failed"; + return -1; + } + if (sparse_file_write(s.get(), tmp.fd, false, false, false) < 0) { + LOG(ERROR) << "sparse_file_write failed"; + return -1; + } + source_image_.reset(tmp.release()); + } else { + source_image_ = std::move(unzip_fd); + } + return source_image_; +} + +template +class ExtentIter final { + public: + ExtentIter(const ContainerType& container) + : iter_(container.cbegin()), end_(container.cend()) {} + + bool GetNext(uint64_t* block) { + while (iter_ != end_) { + if (dst_index_ < iter_->num_blocks()) { + break; + } + iter_++; + dst_index_ = 0; + } + if (iter_ == end_) { + return false; + } + *block = iter_->start_block() + dst_index_; + dst_index_++; + return true; + } + + private: + typename ContainerType::const_iterator iter_; + typename ContainerType::const_iterator end_; + uint64_t dst_index_; +}; + +bool PayloadConverter::ProcessCopy(const InstallOperation& op) { + ExtentIter dst_blocks(op.dst_extents()); + + for (const auto& extent : op.src_extents()) { + for (uint64_t i = 0; i < extent.num_blocks(); i++) { + uint64_t src_block = extent.start_block() + i; + uint64_t dst_block; + if (!dst_blocks.GetNext(&dst_block)) { + LOG(ERROR) << "SOURCE_COPY contained mismatching extents"; + return false; + } + if (src_block == dst_block) continue; + if (!writer_->AddCopy(dst_block, src_block)) { + LOG(ERROR) << "Could not add copy operation"; + return false; + } + } + } + return true; +} + +bool PayloadConverter::ProcessReplace(const InstallOperation& op) { + auto buffer_size = op.data_length(); + auto buffer = std::make_unique(buffer_size); + uint64_t offs = payload_offset_ + op.data_offset(); + if (lseek(in_fd_.get(), offs, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek " << offs << " failed"; + return false; + } + if (!android::base::ReadFully(in_fd_, buffer.get(), buffer_size)) { + PLOG(ERROR) << "read " << buffer_size << " bytes from offset " << offs << "failed"; + return false; + } + + uint64_t dst_size = 0; + for (const auto& extent : op.dst_extents()) { + dst_size += extent.num_blocks() * kBlockSize; + } + + if (op.type() == InstallOperation::REPLACE_BZ) { + auto tmp = std::make_unique(dst_size); + + uint32_t actual_size; + if (dst_size > std::numeric_limits::max()) { + LOG(ERROR) << "too many bytes to decompress: " << dst_size; + return false; + } + actual_size = static_cast(dst_size); + + auto rv = BZ2_bzBuffToBuffDecompress(tmp.get(), &actual_size, buffer.get(), buffer_size, 0, + 0); + if (rv) { + LOG(ERROR) << "bz2 decompress failed: " << rv; + return false; + } + if (actual_size != dst_size) { + LOG(ERROR) << "bz2 returned " << actual_size << " bytes, expected " << dst_size; + return false; + } + buffer = std::move(tmp); + buffer_size = dst_size; + } else if (op.type() == InstallOperation::REPLACE_XZ) { + constexpr uint32_t kXzMaxDictSize = 64 * 1024 * 1024; + + if (dst_size > std::numeric_limits::max()) { + LOG(ERROR) << "too many bytes to decompress: " << dst_size; + return false; + } + + std::unique_ptr s( + xz_dec_init(XZ_DYNALLOC, kXzMaxDictSize), xz_dec_end); + if (!s) { + LOG(ERROR) << "xz_dec_init failed"; + return false; + } + + auto tmp = std::make_unique(dst_size); + + struct xz_buf args; + args.in = reinterpret_cast(buffer.get()); + args.in_pos = 0; + args.in_size = buffer_size; + args.out = reinterpret_cast(tmp.get()); + args.out_pos = 0; + args.out_size = dst_size; + + auto rv = xz_dec_run(s.get(), &args); + if (rv != XZ_STREAM_END) { + LOG(ERROR) << "xz decompress failed: " << (int)rv; + return false; + } + buffer = std::move(tmp); + buffer_size = dst_size; + } + + uint64_t buffer_pos = 0; + for (const auto& extent : op.dst_extents()) { + uint64_t extent_size = extent.num_blocks() * kBlockSize; + if (buffer_size - buffer_pos < extent_size) { + LOG(ERROR) << "replace op ran out of input buffer"; + return false; + } + if (!writer_->AddRawBlocks(extent.start_block(), buffer.get() + buffer_pos, extent_size)) { + LOG(ERROR) << "failed to add raw blocks from replace op"; + return false; + } + buffer_pos += extent_size; + } + return true; +} + +bool PayloadConverter::OpenPayload() { + in_fd_.reset(open(in_file_.c_str(), O_RDONLY)); + if (in_fd_ < 0) { + PLOG(ERROR) << "open " << in_file_; + return false; + } + + char magic[4]; + if (!android::base::ReadFully(in_fd_, magic, sizeof(magic))) { + PLOG(ERROR) << "read magic"; + return false; + } + if (std::string(magic, sizeof(magic)) != "CrAU") { + LOG(ERROR) << "Invalid magic in " << in_file_; + return false; + } + + uint64_t version; + uint64_t manifest_size; + uint32_t manifest_signature_size = 0; + if (!android::base::ReadFully(in_fd_, &version, sizeof(version))) { + PLOG(ERROR) << "read version"; + return false; + } + version = ToLittleEndian(version); + if (version < 2) { + LOG(ERROR) << "Only payload version 2 or higher is supported."; + return false; + } + + if (!android::base::ReadFully(in_fd_, &manifest_size, sizeof(manifest_size))) { + PLOG(ERROR) << "read manifest_size"; + return false; + } + manifest_size = ToLittleEndian(manifest_size); + if (!android::base::ReadFully(in_fd_, &manifest_signature_size, + sizeof(manifest_signature_size))) { + PLOG(ERROR) << "read manifest_signature_size"; + return false; + } + manifest_signature_size = ntohl(manifest_signature_size); + + auto manifest = std::make_unique(manifest_size); + if (!android::base::ReadFully(in_fd_, manifest.get(), manifest_size)) { + PLOG(ERROR) << "read manifest"; + return false; + } + + // Skip past manifest signature. + auto offs = lseek(in_fd_, manifest_signature_size, SEEK_CUR); + if (offs < 0) { + PLOG(ERROR) << "lseek failed"; + return false; + } + payload_offset_ = offs; + + if (!manifest_.ParseFromArray(manifest.get(), manifest_size)) { + LOG(ERROR) << "could not parse manifest"; + return false; + } + return true; +} + +} // namespace snapshot +} // namespace android + +int main(int argc, char** argv) { + android::base::InitLogging(argv, android::snapshot::MyLogger); + gflags::SetUsageMessage("Convert OTA payload to a Virtual A/B COW"); + int arg_start = gflags::ParseCommandLineFlags(&argc, &argv, false); + + xz_crc32_init(); + + if (argc - arg_start != 2) { + std::cerr << "Usage: [options] \n"; + return 1; + } + + android::snapshot::PayloadConverter pc(argv[arg_start], argv[arg_start + 1]); + return pc.Run() ? 0 : 1; +}