diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index 95fbab8ba..0bb1b8719 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -412,6 +412,7 @@ cc_test { test_min_api_level: 30, auto_gen_config: true, require_root: false, + host_supported: true, } cc_binary { diff --git a/fs_mgr/libsnapshot/cow_api_test.cpp b/fs_mgr/libsnapshot/cow_api_test.cpp index ec206ad9a..40d5efbeb 100644 --- a/fs_mgr/libsnapshot/cow_api_test.cpp +++ b/fs_mgr/libsnapshot/cow_api_test.cpp @@ -70,7 +70,7 @@ TEST_F(CowTest, ReadWrite) { ASSERT_TRUE(writer.AddCopy(10, 20)); ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); ASSERT_TRUE(writer.AddZeroBlocks(51, 2)); - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); @@ -105,7 +105,7 @@ TEST_F(CowTest, ReadWrite) { ASSERT_EQ(op->compression, kCowCompressNone); ASSERT_EQ(op->data_length, 4096); ASSERT_EQ(op->new_block, 50); - ASSERT_EQ(op->source, 104); + ASSERT_EQ(op->source, 106); ASSERT_TRUE(reader.ReadData(*op, &sink)); ASSERT_EQ(sink.stream(), data); @@ -145,7 +145,7 @@ TEST_F(CowTest, CompressGz) { data.resize(options.block_size, '\0'); ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); @@ -163,7 +163,7 @@ TEST_F(CowTest, CompressGz) { ASSERT_EQ(op->compression, kCowCompressGz); ASSERT_EQ(op->data_length, 56); // compressed! ASSERT_EQ(op->new_block, 50); - ASSERT_EQ(op->source, 104); + ASSERT_EQ(op->source, 106); ASSERT_TRUE(reader.ReadData(*op, &sink)); ASSERT_EQ(sink.stream(), data); @@ -182,7 +182,7 @@ TEST_F(CowTest, CompressTwoBlocks) { data.resize(options.block_size * 2, '\0'); ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); @@ -224,7 +224,7 @@ TEST_P(CompressionTest, HorribleSink) { data.resize(options.block_size, '\0'); ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); @@ -259,7 +259,7 @@ TEST_F(CowTest, GetSize) { ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); ASSERT_TRUE(writer.AddZeroBlocks(51, 2)); auto size_before = writer.GetCowSize(); - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); auto size_after = writer.GetCowSize(); ASSERT_EQ(size_before, size_after); struct stat buf; @@ -271,6 +271,60 @@ TEST_F(CowTest, GetSize) { ASSERT_EQ(buf.st_size, writer.GetCowSize()); } +TEST_F(CowTest, Append) { + CowOptions options; + auto writer = std::make_unique(options); + ASSERT_TRUE(writer->Initialize(cow_->fd)); + + std::string data = "This is some data, believe it"; + data.resize(options.block_size, '\0'); + ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size())); + ASSERT_TRUE(writer->Flush()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + writer = std::make_unique(options); + ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND)); + + std::string data2 = "More data!"; + data2.resize(options.block_size, '\0'); + ASSERT_TRUE(writer->AddRawBlocks(51, data2.data(), data2.size())); + ASSERT_TRUE(writer->Flush()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + struct stat buf; + ASSERT_EQ(fstat(cow_->fd, &buf), 0); + ASSERT_EQ(buf.st_size, writer->GetCowSize()); + + // Read back both operations. + CowReader reader; + ASSERT_TRUE(reader.Parse(cow_->fd)); + + StringSink sink; + + auto iter = reader.GetOpIter(); + ASSERT_NE(iter, nullptr); + + ASSERT_FALSE(iter->Done()); + auto op = &iter->Get(); + ASSERT_EQ(op->type, kCowReplaceOp); + ASSERT_TRUE(reader.ReadData(*op, &sink)); + ASSERT_EQ(sink.stream(), data); + + iter->Next(); + sink.Reset(); + + ASSERT_FALSE(iter->Done()); + op = &iter->Get(); + ASSERT_EQ(op->type, kCowReplaceOp); + ASSERT_TRUE(reader.ReadData(*op, &sink)); + ASSERT_EQ(sink.stream(), data2); + + iter->Next(); + ASSERT_TRUE(iter->Done()); +} + } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/cow_reader.cpp b/fs_mgr/libsnapshot/cow_reader.cpp index 720ee3b45..60093ab68 100644 --- a/fs_mgr/libsnapshot/cow_reader.cpp +++ b/fs_mgr/libsnapshot/cow_reader.cpp @@ -78,6 +78,11 @@ bool CowReader::Parse(android::base::borrowed_fd fd) { << "Expected: " << kCowMagicNumber; return false; } + if (header_.header_size != sizeof(CowHeader)) { + LOG(ERROR) << "Header size unknown, read " << header_.header_size << ", expected " + << sizeof(CowHeader); + return false; + } if ((header_.major_version != kCowVersionMajor) || (header_.minor_version != kCowVersionMinor)) { diff --git a/fs_mgr/libsnapshot/cow_snapuserd_test.cpp b/fs_mgr/libsnapshot/cow_snapuserd_test.cpp index d767022fb..4c8ff5f2a 100644 --- a/fs_mgr/libsnapshot/cow_snapuserd_test.cpp +++ b/fs_mgr/libsnapshot/cow_snapuserd_test.cpp @@ -128,7 +128,7 @@ TEST_F(SnapuserdTest, ReadWrite) { ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_2.get(), size)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); diff --git a/fs_mgr/libsnapshot/cow_writer.cpp b/fs_mgr/libsnapshot/cow_writer.cpp index f05f9ba4b..4cf2119a7 100644 --- a/fs_mgr/libsnapshot/cow_writer.cpp +++ b/fs_mgr/libsnapshot/cow_writer.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -40,17 +41,48 @@ void CowWriter::SetupHeaders() { header_.magic = kCowMagicNumber; header_.major_version = kCowVersionMajor; header_.minor_version = kCowVersionMinor; + header_.header_size = sizeof(CowHeader); header_.block_size = options_.block_size; } -bool CowWriter::Initialize(android::base::unique_fd&& fd) { - owned_fd_ = std::move(fd); - return Initialize(android::base::borrowed_fd{owned_fd_}); +bool CowWriter::ParseOptions() { + if (options_.compression == "gz") { + compression_ = kCowCompressGz; + } else if (options_.compression == "brotli") { + compression_ = kCowCompressBrotli; + } else if (options_.compression == "none") { + compression_ = kCowCompressNone; + } else if (!options_.compression.empty()) { + LOG(ERROR) << "unrecognized compression: " << options_.compression; + return false; + } + return true; } -bool CowWriter::Initialize(android::base::borrowed_fd fd) { +bool CowWriter::Initialize(android::base::unique_fd&& fd, OpenMode mode) { + owned_fd_ = std::move(fd); + return Initialize(android::base::borrowed_fd{owned_fd_}, mode); +} + +bool CowWriter::Initialize(android::base::borrowed_fd fd, OpenMode mode) { fd_ = fd; + if (!ParseOptions()) { + return false; + } + + switch (mode) { + case OpenMode::WRITE: + return OpenForWrite(); + case OpenMode::APPEND: + return OpenForAppend(); + default: + LOG(ERROR) << "Unknown open mode in CowWriter"; + return false; + } +} + +bool CowWriter::OpenForWrite() { // This limitation is tied to the data field size in CowOperation. if (header_.block_size > std::numeric_limits::max()) { LOG(ERROR) << "Block size is too large"; @@ -62,35 +94,52 @@ bool CowWriter::Initialize(android::base::borrowed_fd fd) { return false; } - if (options_.compression == "gz") { - compression_ = kCowCompressGz; - } else if (options_.compression == "brotli") { - compression_ = kCowCompressBrotli; - } else if (options_.compression == "none") { - compression_ = kCowCompressNone; - } else if (!options_.compression.empty()) { - LOG(ERROR) << "unrecognized compression: " << options_.compression; + // Headers are not complete, but this ensures the file is at the right + // position. + if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) { + PLOG(ERROR) << "write failed"; return false; } - // Headers are not complete, but this ensures the file is at the right - // position. - if (!WriteFully(fd_, &header_, sizeof(header_))) { - PLOG(ERROR) << "write failed"; + header_.ops_offset = header_.header_size; + return true; +} + +bool CowWriter::OpenForAppend() { + auto reader = std::make_unique(); + if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) { + return false; + } + options_.block_size = header_.block_size; + + // Reset this, since we're going to reimport all operations. + header_.num_ops = 0; + + auto iter = reader->GetOpIter(); + while (!iter->Done()) { + auto& op = iter->Get(); + AddOperation(op); + + iter->Next(); + } + + // Free reader so we own the descriptor position again. + reader = nullptr; + + // Seek to the end of the data section. + if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek failed"; return false; } return true; } bool CowWriter::AddCopy(uint64_t new_block, uint64_t old_block) { - header_.num_ops++; - CowOperation op = {}; op.type = kCowCopyOp; op.new_block = new_block; op.source = old_block; - ops_ += std::basic_string(reinterpret_cast(&op), sizeof(op)); - + AddOperation(op); return true; } @@ -108,8 +157,6 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t const uint8_t* iter = reinterpret_cast(data); for (size_t i = 0; i < size / header_.block_size; i++) { - header_.num_ops++; - CowOperation op = {}; op.type = kCowReplaceOp; op.new_block = new_block_start + i; @@ -125,7 +172,7 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t LOG(ERROR) << "Compressed block is too large: " << data.size() << " bytes"; return false; } - if (!WriteFully(fd_, data.data(), data.size())) { + if (!WriteRawData(data.data(), data.size())) { PLOG(ERROR) << "AddRawBlocks: write failed"; return false; } @@ -137,11 +184,11 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t pos += header_.block_size; } - ops_ += std::basic_string(reinterpret_cast(&op), sizeof(op)); + AddOperation(op); iter += header_.block_size; } - if (!compression_ && !WriteFully(fd_, data, size)) { + if (!compression_ && !WriteRawData(data, size)) { PLOG(ERROR) << "AddRawBlocks: write failed"; return false; } @@ -150,13 +197,11 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t bool CowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { for (uint64_t i = 0; i < num_blocks; i++) { - header_.num_ops++; - CowOperation op = {}; op.type = kCowZeroOp; op.new_block = new_block_start + i; op.source = 0; - ops_ += std::basic_string(reinterpret_cast(&op), sizeof(op)); + AddOperation(op); } return true; } @@ -212,17 +257,7 @@ static void SHA256(const void*, size_t, uint8_t[]) { #endif } -bool CowWriter::Finalize() { - // If both fields are set then Finalize is already called. - if (header_.ops_offset > 0 && header_.ops_size > 0) { - return true; - } - auto offs = lseek(fd_.get(), 0, SEEK_CUR); - if (offs < 0) { - PLOG(ERROR) << "lseek failed"; - return false; - } - header_.ops_offset = offs; +bool CowWriter::Flush() { header_.ops_size = ops_.size(); memset(header_.ops_checksum, 0, sizeof(uint8_t) * 32); @@ -235,8 +270,6 @@ bool CowWriter::Finalize() { PLOG(ERROR) << "lseek failed"; return false; } - // Header is already written, calling WriteFully will increment - // bytes_written_. So use android::base::WriteFully() here. if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) { PLOG(ERROR) << "write header failed"; return false; @@ -250,13 +283,16 @@ bool CowWriter::Finalize() { return false; } - // clear ops_ so that subsequent calls to GetSize() still works. - ops_.clear(); + // Re-position for any subsequent writes. + if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek ops failed"; + return false; + } return true; } size_t CowWriter::GetCowSize() { - return bytes_written_ + ops_.size() * sizeof(ops_[0]); + return header_.ops_offset + header_.num_ops * sizeof(CowOperation); } bool CowWriter::GetDataPos(uint64_t* pos) { @@ -269,9 +305,17 @@ bool CowWriter::GetDataPos(uint64_t* pos) { return true; } -bool CowWriter::WriteFully(base::borrowed_fd fd, const void* data, size_t size) { - bytes_written_ += size; - return android::base::WriteFully(fd, data, size); +void CowWriter::AddOperation(const CowOperation& op) { + header_.num_ops++; + ops_.insert(ops_.size(), reinterpret_cast(&op), sizeof(op)); +} + +bool CowWriter::WriteRawData(const void* data, size_t size) { + if (!android::base::WriteFully(fd_, data, size)) { + return false; + } + header_.ops_offset += size; + return true; } } // namespace snapshot diff --git a/fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp b/fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp index 45833e121..2a0136b6d 100644 --- a/fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp +++ b/fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp @@ -375,7 +375,7 @@ bool NonAbEstimator::AnalyzePartition(const std::string& partition_name) { } } - if (!writer->Finalize()) { + if (!writer->Flush()) { return false; } diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h index 0adce4867..4a6bd4ed0 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h @@ -45,6 +45,9 @@ struct CowHeader { uint16_t major_version; uint16_t minor_version; + // Size of this struct. + uint16_t header_size; + // Offset to the location of the operation sequence, and size of the // operation sequence buffer. |ops_offset| is also the end of the // raw data region. diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h index 8826b7a02..8569161fb 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h @@ -47,12 +47,11 @@ class ICowWriter { // Encode a sequence of zeroed blocks. |size| must be a multiple of the block size. virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0; - // Finalize all COW operations and flush pending writes. - // Return true if successful. - virtual bool Finalize() = 0; + // Flush all pending writes. This must be called before closing the writer + // to ensure that the correct headers and footers are written. + virtual bool Flush() = 0; - // Return 0 if failed, on success return number of bytes the cow image would be - // after calling Finalize(); + // Return number of bytes the cow image occupies on disk. virtual size_t GetCowSize() = 0; protected: @@ -61,24 +60,30 @@ class ICowWriter { class CowWriter : public ICowWriter { public: + enum class OpenMode { WRITE, APPEND }; + explicit CowWriter(const CowOptions& options); // Set up the writer. - bool Initialize(android::base::unique_fd&& fd); - bool Initialize(android::base::borrowed_fd fd); + bool Initialize(android::base::unique_fd&& fd, OpenMode mode = OpenMode::WRITE); + bool Initialize(android::base::borrowed_fd fd, OpenMode mode = OpenMode::WRITE); bool AddCopy(uint64_t new_block, uint64_t old_block) override; bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; - bool Finalize() override; + bool Flush() override; size_t GetCowSize() override; private: void SetupHeaders(); + bool ParseOptions(); + bool OpenForWrite(); + bool OpenForAppend(); bool GetDataPos(uint64_t* pos); - bool WriteFully(base::borrowed_fd fd, const void* data, size_t size); + bool WriteRawData(const void* data, size_t size); + void AddOperation(const CowOperation& op); std::basic_string Compress(const void* data, size_t length); private: @@ -90,7 +95,6 @@ class CowWriter : public ICowWriter { // :TODO: this is not efficient, but stringstream ubsan aborts because some // bytes overflow a signed char. std::basic_string ops_; - std::atomic bytes_written_ = 0; }; } // namespace snapshot diff --git a/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp b/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp index d0b8f52e0..f76107709 100644 --- a/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp +++ b/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp @@ -204,7 +204,7 @@ bool PayloadConverter::ProcessPartition(const PartitionUpdate& update) { } } - if (!writer_->Finalize()) { + if (!writer_->Flush()) { LOG(ERROR) << "Unable to finalize COW for " << partition_name; return false; }