diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index b239f3155..3d57b59e5 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -149,6 +149,7 @@ cc_defaults { "cow_decompress.cpp", "cow_reader.cpp", "cow_writer.cpp", + "cow_format.cpp", ], } diff --git a/fs_mgr/libsnapshot/cow_api_test.cpp b/fs_mgr/libsnapshot/cow_api_test.cpp index f9e9b4ca7..ffee52fd1 100644 --- a/fs_mgr/libsnapshot/cow_api_test.cpp +++ b/fs_mgr/libsnapshot/cow_api_test.cpp @@ -76,13 +76,15 @@ TEST_F(CowTest, ReadWrite) { CowReader reader; CowHeader header; + CowFooter footer; ASSERT_TRUE(reader.Parse(cow_->fd)); ASSERT_TRUE(reader.GetHeader(&header)); + ASSERT_TRUE(reader.GetFooter(&footer)); ASSERT_EQ(header.magic, kCowMagicNumber); ASSERT_EQ(header.major_version, kCowVersionMajor); ASSERT_EQ(header.minor_version, kCowVersionMinor); ASSERT_EQ(header.block_size, options.block_size); - ASSERT_EQ(header.num_ops, 4); + ASSERT_EQ(footer.op.num_ops, 4); auto iter = reader.GetOpIter(); ASSERT_NE(iter, nullptr); @@ -105,7 +107,6 @@ TEST_F(CowTest, ReadWrite) { ASSERT_EQ(op->compression, kCowCompressNone); ASSERT_EQ(op->data_length, 4096); ASSERT_EQ(op->new_block, 50); - ASSERT_EQ(op->source, 106); ASSERT_TRUE(reader.ReadData(*op, &sink)); ASSERT_EQ(sink.stream(), data); @@ -163,7 +164,6 @@ TEST_F(CowTest, CompressGz) { ASSERT_EQ(op->compression, kCowCompressGz); ASSERT_EQ(op->data_length, 56); // compressed! ASSERT_EQ(op->new_block, 50); - ASSERT_EQ(op->source, 106); ASSERT_TRUE(reader.ReadData(*op, &sink)); ASSERT_EQ(sink.stream(), data); @@ -272,6 +272,7 @@ TEST_F(CowTest, GetSize) { } TEST_F(CowTest, Append) { + cow_->DoNotRemove(); CowOptions options; auto writer = std::make_unique(options); ASSERT_TRUE(writer->Initialize(cow_->fd)); @@ -325,6 +326,139 @@ TEST_F(CowTest, Append) { ASSERT_TRUE(iter->Done()); } +TEST_F(CowTest, AppendCorrupted) { + CowOptions options; + auto writer = std::make_unique(options); + ASSERT_TRUE(writer->Initialize(cow_->fd)); + + std::string data = "This is some data, believe it"; + data.resize(options.block_size, '\0'); + ASSERT_TRUE(writer->AddLabel(0)); + ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size())); + ASSERT_TRUE(writer->AddLabel(1)); + ASSERT_TRUE(writer->AddZeroBlocks(50, 1)); + ASSERT_TRUE(writer->Finalize()); + // Drop the tail end of the header. Last entry may be corrupted. + ftruncate(cow_->fd, writer->GetCowSize() - 5); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + writer = std::make_unique(options); + ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND)); + + ASSERT_TRUE(writer->AddLabel(2)); + ASSERT_TRUE(writer->AddZeroBlocks(50, 1)); + + std::string data2 = "More data!"; + data2.resize(options.block_size, '\0'); + ASSERT_TRUE(writer->AddLabel(3)); + ASSERT_TRUE(writer->AddRawBlocks(51, data2.data(), data2.size())); + ASSERT_TRUE(writer->Finalize()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + struct stat buf; + ASSERT_EQ(fstat(cow_->fd, &buf), 0); + ASSERT_EQ(buf.st_size, writer->GetCowSize()); + + // Read back all three operations. + CowReader reader; + ASSERT_TRUE(reader.Parse(cow_->fd)); + + StringSink sink; + + auto iter = reader.GetOpIter(); + ASSERT_NE(iter, nullptr); + + ASSERT_FALSE(iter->Done()); + auto op = &iter->Get(); + ASSERT_EQ(op->type, kCowLabelOp); + ASSERT_EQ(op->source, 0); + + iter->Next(); + + ASSERT_FALSE(iter->Done()); + op = &iter->Get(); + ASSERT_EQ(op->type, kCowReplaceOp); + ASSERT_TRUE(reader.ReadData(*op, &sink)); + ASSERT_EQ(sink.stream(), data); + + iter->Next(); + sink.Reset(); + + ASSERT_FALSE(iter->Done()); + op = &iter->Get(); + ASSERT_EQ(op->type, kCowLabelOp); + ASSERT_EQ(op->source, 2); + + iter->Next(); + + ASSERT_FALSE(iter->Done()); + op = &iter->Get(); + ASSERT_EQ(op->type, kCowZeroOp); + + iter->Next(); + + ASSERT_FALSE(iter->Done()); + op = &iter->Get(); + ASSERT_EQ(op->type, kCowLabelOp); + ASSERT_EQ(op->source, 3); + + iter->Next(); + + ASSERT_FALSE(iter->Done()); + op = &iter->Get(); + ASSERT_EQ(op->type, kCowReplaceOp); + ASSERT_TRUE(reader.ReadData(*op, &sink)); + ASSERT_EQ(sink.stream(), data2); + + iter->Next(); + ASSERT_TRUE(iter->Done()); +} + +TEST_F(CowTest, AppendExtendedCorrupted) { + CowOptions options; + auto writer = std::make_unique(options); + ASSERT_TRUE(writer->Initialize(cow_->fd)); + + ASSERT_TRUE(writer->AddLabel(5)); + ASSERT_TRUE(writer->AddLabel(6)); + + std::string data = "This is some data, believe it"; + data.resize(options.block_size * 2, '\0'); + ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size())); + + // fail to write the footer + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + writer = std::make_unique(options); + ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND)); + + ASSERT_TRUE(writer->Finalize()); + + struct stat buf; + ASSERT_EQ(fstat(cow_->fd, &buf), 0); + ASSERT_EQ(buf.st_size, writer->GetCowSize()); + + // Read back all three operations. + CowReader reader; + ASSERT_TRUE(reader.Parse(cow_->fd)); + + StringSink sink; + + auto iter = reader.GetOpIter(); + ASSERT_NE(iter, nullptr); + + ASSERT_FALSE(iter->Done()); + auto op = &iter->Get(); + ASSERT_EQ(op->type, kCowLabelOp); + ASSERT_EQ(op->source, 5); + + iter->Next(); + ASSERT_TRUE(iter->Done()); +} + } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/cow_format.cpp b/fs_mgr/libsnapshot/cow_format.cpp new file mode 100644 index 000000000..49ba11f7c --- /dev/null +++ b/fs_mgr/libsnapshot/cow_format.cpp @@ -0,0 +1,63 @@ +// +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include +#include + +#include + +namespace android { +namespace snapshot { + +std::ostream& operator<<(std::ostream& os, CowOperation const& op) { + os << "CowOperation(type:"; + if (op.type == kCowCopyOp) + os << "kCowCopyOp, "; + else if (op.type == kCowReplaceOp) + os << "kCowReplaceOp, "; + else if (op.type == kCowZeroOp) + os << "kZeroOp, "; + else if (op.type == kCowFooterOp) + os << "kCowFooterOp, "; + else if (op.type == kCowLabelOp) + os << "kCowLabelOp, "; + else + os << (int)op.type << "?,"; + os << "compression:"; + if (op.compression == kCowCompressNone) + os << "kCowCompressNone, "; + else if (op.compression == kCowCompressGz) + os << "kCowCompressGz, "; + else if (op.compression == kCowCompressBrotli) + os << "kCowCompressBrotli, "; + else + os << (int)op.compression << "?, "; + os << "data_length:" << op.data_length << ",\t"; + os << "new_block:" << op.new_block << ",\t"; + os << "source:" << op.source << ")"; + return os; +} + +int64_t GetNextOpOffset(const CowOperation& op) { + if (op.type == kCowReplaceOp) + return op.data_length; + else + return 0; +} + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/cow_reader.cpp b/fs_mgr/libsnapshot/cow_reader.cpp index 60093ab68..6d6a22d86 100644 --- a/fs_mgr/libsnapshot/cow_reader.cpp +++ b/fs_mgr/libsnapshot/cow_reader.cpp @@ -18,17 +18,19 @@ #include #include +#include #include #include #include #include + #include "cow_decompress.h" namespace android { namespace snapshot { -CowReader::CowReader() : fd_(-1), header_(), fd_size_(0) {} +CowReader::CowReader() : fd_(-1), header_(), footer_(), fd_size_(0), has_footer_(false) {} static void SHA256(const void*, size_t, uint8_t[]) { #if 0 @@ -63,16 +65,6 @@ bool CowReader::Parse(android::base::borrowed_fd fd) { return false; } - // Validity check the ops range. - if (header_.ops_offset >= fd_size_) { - LOG(ERROR) << "ops offset " << header_.ops_offset << " larger than fd size " << fd_size_; - return false; - } - if (fd_size_ - header_.ops_offset < header_.ops_size) { - LOG(ERROR) << "ops size " << header_.ops_size << " is too large"; - return false; - } - if (header_.magic != kCowMagicNumber) { LOG(ERROR) << "Header Magic corrupted. Magic: " << header_.magic << "Expected: " << kCowMagicNumber; @@ -83,6 +75,11 @@ bool CowReader::Parse(android::base::borrowed_fd fd) { << sizeof(CowHeader); return false; } + if (header_.footer_size != sizeof(CowFooter)) { + LOG(ERROR) << "Footer size unknown, read " << header_.footer_size << ", expected " + << sizeof(CowFooter); + return false; + } if ((header_.major_version != kCowVersionMajor) || (header_.minor_version != kCowVersionMinor)) { @@ -94,19 +91,16 @@ bool CowReader::Parse(android::base::borrowed_fd fd) { return false; } - uint8_t header_csum[32]; - { - CowHeader tmp = header_; - memset(&tmp.header_checksum, 0, sizeof(tmp.header_checksum)); - memset(header_csum, 0, sizeof(uint8_t) * 32); - - SHA256(&tmp, sizeof(tmp), header_csum); - } - if (memcmp(header_csum, header_.header_checksum, sizeof(header_csum)) != 0) { - LOG(ERROR) << "header checksum is invalid"; + auto footer_pos = lseek(fd_.get(), -header_.footer_size, SEEK_END); + if (footer_pos != fd_size_ - header_.footer_size) { + LOG(ERROR) << "Failed to read full footer!"; return false; } - + if (!android::base::ReadFully(fd_, &footer_, sizeof(footer_))) { + PLOG(ERROR) << "read footer failed"; + return false; + } + has_footer_ = (footer_.op.type == kCowFooterOp); return true; } @@ -115,74 +109,88 @@ bool CowReader::GetHeader(CowHeader* header) { return true; } +bool CowReader::GetFooter(CowFooter* footer) { + if (!has_footer_) return false; + *footer = footer_; + return true; +} + class CowOpIter final : public ICowOpIter { public: - CowOpIter(std::unique_ptr&& ops, size_t len); + CowOpIter(std::unique_ptr>&& ops); bool Done() override; const CowOperation& Get() override; void Next() override; private: - bool HasNext(); - - std::unique_ptr ops_; - const uint8_t* pos_; - const uint8_t* end_; - bool done_; + std::unique_ptr> ops_; + std::vector::iterator op_iter_; }; -CowOpIter::CowOpIter(std::unique_ptr&& ops, size_t len) - : ops_(std::move(ops)), pos_(ops_.get()), end_(pos_ + len), done_(!HasNext()) {} - -bool CowOpIter::Done() { - return done_; +CowOpIter::CowOpIter(std::unique_ptr>&& ops) { + ops_ = std::move(ops); + op_iter_ = ops_.get()->begin(); } -bool CowOpIter::HasNext() { - return pos_ < end_ && size_t(end_ - pos_) >= sizeof(CowOperation); +bool CowOpIter::Done() { + return op_iter_ == ops_.get()->end(); } void CowOpIter::Next() { CHECK(!Done()); - - pos_ += sizeof(CowOperation); - if (!HasNext()) done_ = true; + op_iter_++; } const CowOperation& CowOpIter::Get() { CHECK(!Done()); - CHECK(HasNext()); - return *reinterpret_cast(pos_); + return (*op_iter_); } std::unique_ptr CowReader::GetOpIter() { - if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) { + uint64_t pos = lseek(fd_.get(), sizeof(header_), SEEK_SET); + if (pos != sizeof(header_)) { PLOG(ERROR) << "lseek ops failed"; return nullptr; } - auto ops_buffer = std::make_unique(header_.ops_size); - if (!android::base::ReadFully(fd_, ops_buffer.get(), header_.ops_size)) { - PLOG(ERROR) << "read ops failed"; - return nullptr; + auto ops_buffer = std::make_unique>(); + if (has_footer_) ops_buffer->reserve(footer_.op.num_ops); + uint64_t current_op_num = 0; + uint64_t last_pos = fd_size_ - (has_footer_ ? sizeof(footer_) : sizeof(CowOperation)); + + // Alternating op and data + while (pos < last_pos) { + ops_buffer->resize(current_op_num + 1); + if (!android::base::ReadFully(fd_, ops_buffer->data() + current_op_num, + sizeof(CowOperation))) { + PLOG(ERROR) << "read op failed"; + return nullptr; + } + auto current_op = ops_buffer->data()[current_op_num]; + pos = lseek(fd_.get(), GetNextOpOffset(current_op), SEEK_CUR); + current_op_num++; } uint8_t csum[32]; memset(csum, 0, sizeof(uint8_t) * 32); - SHA256(ops_buffer.get(), header_.ops_size, csum); - if (memcmp(csum, header_.ops_checksum, sizeof(csum)) != 0) { - LOG(ERROR) << "ops checksum does not match"; - return nullptr; + if (has_footer_) { + SHA256(ops_buffer.get()->data(), footer_.op.ops_size, csum); + if (memcmp(csum, footer_.data.ops_checksum, sizeof(csum)) != 0) { + LOG(ERROR) << "ops checksum does not match"; + return nullptr; + } + } else { + LOG(INFO) << "No Footer, recovered data"; } - return std::make_unique(std::move(ops_buffer), header_.ops_size); + return std::make_unique(std::move(ops_buffer)); } bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) { // Validate the offset, taking care to acknowledge possible overflow of offset+len. - if (offset < sizeof(header_) || offset >= header_.ops_offset || len >= fd_size_ || - offset + len > header_.ops_offset) { + if (offset < sizeof(header_) || offset >= fd_size_ - sizeof(footer_) || len >= fd_size_ || + offset + len > fd_size_ - sizeof(footer_)) { LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes"; return false; } diff --git a/fs_mgr/libsnapshot/cow_writer.cpp b/fs_mgr/libsnapshot/cow_writer.cpp index f9ba0b3c2..679b55e61 100644 --- a/fs_mgr/libsnapshot/cow_writer.cpp +++ b/fs_mgr/libsnapshot/cow_writer.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -65,6 +66,10 @@ bool ICowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { return EmitZeroBlocks(new_block_start, num_blocks); } +bool ICowWriter::AddLabel(uint64_t label) { + return EmitLabel(label); +} + bool ICowWriter::ValidateNewBlock(uint64_t new_block) { if (options_.max_blocks && new_block >= options_.max_blocks.value()) { LOG(ERROR) << "New block " << new_block << " exceeds maximum block count " @@ -84,7 +89,11 @@ void CowWriter::SetupHeaders() { header_.major_version = kCowVersionMajor; header_.minor_version = kCowVersionMinor; header_.header_size = sizeof(CowHeader); + header_.footer_size = sizeof(CowFooter); header_.block_size = options_.block_size; + footer_ = {}; + footer_.op.data_length = 64; + footer_.op.type = kCowFooterOp; } bool CowWriter::ParseOptions() { @@ -143,33 +152,53 @@ bool CowWriter::OpenForWrite() { return false; } - header_.ops_offset = header_.header_size; + next_op_pos_ = sizeof(header_); return true; } bool CowWriter::OpenForAppend() { auto reader = std::make_unique(); + bool incomplete = false; + std::queue toAdd; if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) { return false; } + incomplete = !reader->GetFooter(&footer_); + options_.block_size = header_.block_size; // Reset this, since we're going to reimport all operations. - header_.num_ops = 0; + footer_.op.num_ops = 0; + next_op_pos_ = sizeof(header_); auto iter = reader->GetOpIter(); while (!iter->Done()) { - auto& op = iter->Get(); - AddOperation(op); - + CowOperation op = iter->Get(); + if (op.type == kCowFooterOp) break; + if (incomplete) { + // Last operation translation may be corrupt. Wait to add it. + if (op.type == kCowLabelOp) { + while (!toAdd.empty()) { + AddOperation(toAdd.front()); + toAdd.pop(); + } + } + toAdd.push(op); + } else { + AddOperation(op); + } iter->Next(); } // Free reader so we own the descriptor position again. reader = nullptr; - // Seek to the end of the data section. - if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) { + // Position for new writing + if (ftruncate(fd_.get(), next_op_pos_) != 0) { + PLOG(ERROR) << "Failed to trim file"; + return false; + } + if (lseek(fd_.get(), 0, SEEK_END) < 0) { PLOG(ERROR) << "lseek failed"; return false; } @@ -181,22 +210,18 @@ bool CowWriter::EmitCopy(uint64_t new_block, uint64_t old_block) { op.type = kCowCopyOp; op.new_block = new_block; op.source = old_block; - AddOperation(op); - return true; + return WriteOperation(op); } bool CowWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) { - uint64_t pos; - if (!GetDataPos(&pos)) { - return false; - } - const uint8_t* iter = reinterpret_cast(data); + uint64_t pos; for (size_t i = 0; i < size / header_.block_size; i++) { CowOperation op = {}; op.type = kCowReplaceOp; op.new_block = new_block_start + i; - op.source = pos; + GetDataPos(&pos); + op.source = pos + sizeof(op); if (compression_) { auto data = Compress(iter, header_.block_size); @@ -208,26 +233,23 @@ bool CowWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t LOG(ERROR) << "Compressed block is too large: " << data.size() << " bytes"; return false; } - if (!WriteRawData(data.data(), data.size())) { + op.compression = compression_; + op.data_length = static_cast(data.size()); + + if (!WriteOperation(op, data.data(), data.size())) { PLOG(ERROR) << "AddRawBlocks: write failed"; return false; } - op.compression = compression_; - op.data_length = static_cast(data.size()); - pos += data.size(); } else { op.data_length = static_cast(header_.block_size); - pos += header_.block_size; + if (!WriteOperation(op, iter, header_.block_size)) { + PLOG(ERROR) << "AddRawBlocks: write failed"; + return false; + } } - AddOperation(op); iter += header_.block_size; } - - if (!compression_ && !WriteRawData(data, size)) { - PLOG(ERROR) << "AddRawBlocks: write failed"; - return false; - } return true; } @@ -237,11 +259,18 @@ bool CowWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { op.type = kCowZeroOp; op.new_block = new_block_start + i; op.source = 0; - AddOperation(op); + WriteOperation(op); } return true; } +bool CowWriter::EmitLabel(uint64_t label) { + CowOperation op = {}; + op.type = kCowLabelOp; + op.source = label; + return WriteOperation(op); +} + std::basic_string CowWriter::Compress(const void* data, size_t length) { switch (compression_) { case kCowCompressGz: { @@ -294,33 +323,27 @@ static void SHA256(const void*, size_t, uint8_t[]) { } bool CowWriter::Finalize() { - header_.ops_size = ops_.size(); + footer_.op.ops_size = ops_.size() + sizeof(footer_.op); + uint64_t pos; - memset(header_.ops_checksum, 0, sizeof(uint8_t) * 32); - memset(header_.header_checksum, 0, sizeof(uint8_t) * 32); - - SHA256(ops_.data(), ops_.size(), header_.ops_checksum); - SHA256(&header_, sizeof(header_), header_.header_checksum); - - if (lseek(fd_.get(), 0, SEEK_SET)) { - PLOG(ERROR) << "lseek failed"; + if (!GetDataPos(&pos)) { + PLOG(ERROR) << "failed to get file position"; return false; } - if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) { - PLOG(ERROR) << "write header failed"; - return false; - } - if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) { - PLOG(ERROR) << "lseek ops failed"; - return false; - } - if (!WriteFully(fd_, ops_.data(), ops_.size())) { - PLOG(ERROR) << "write ops failed"; + memset(&footer_.data.ops_checksum, 0, sizeof(uint8_t) * 32); + memset(&footer_.data.footer_checksum, 0, sizeof(uint8_t) * 32); + + SHA256(ops_.data(), ops_.size(), footer_.data.ops_checksum); + SHA256(&footer_.op, sizeof(footer_.op), footer_.data.footer_checksum); + // Write out footer at end of file + if (!android::base::WriteFully(fd_, reinterpret_cast(&footer_), + sizeof(footer_))) { + PLOG(ERROR) << "write footer failed"; return false; } // Re-position for any subsequent writes. - if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) { + if (lseek(fd_.get(), pos, SEEK_SET) < 0) { PLOG(ERROR) << "lseek ops failed"; return false; } @@ -328,7 +351,7 @@ bool CowWriter::Finalize() { } uint64_t CowWriter::GetCowSize() { - return header_.ops_offset + header_.num_ops * sizeof(CowOperation); + return next_op_pos_ + sizeof(footer_); } bool CowWriter::GetDataPos(uint64_t* pos) { @@ -341,8 +364,19 @@ bool CowWriter::GetDataPos(uint64_t* pos) { return true; } +bool CowWriter::WriteOperation(const CowOperation& op, const void* data, size_t size) { + if (!android::base::WriteFully(fd_, reinterpret_cast(&op), sizeof(op))) { + return false; + } + if (data != NULL && size > 0) + if (!WriteRawData(data, size)) return false; + AddOperation(op); + return !fsync(fd_.get()); +} + void CowWriter::AddOperation(const CowOperation& op) { - header_.num_ops++; + footer_.op.num_ops++; + next_op_pos_ += sizeof(CowOperation) + GetNextOpOffset(op); ops_.insert(ops_.size(), reinterpret_cast(&op), sizeof(op)); } @@ -350,7 +384,6 @@ bool CowWriter::WriteRawData(const void* data, size_t size) { if (!android::base::WriteFully(fd_, data, size)) { return false; } - header_.ops_offset += size; return true; } diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h index 4a6bd4ed0..2291e309c 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h @@ -15,6 +15,7 @@ #pragma once #include +#include namespace android { namespace snapshot { @@ -29,17 +30,22 @@ static constexpr uint32_t kCowVersionMinor = 0; // +-----------------------+ // | Header (fixed) | // +-----------------------+ -// | Raw Data (variable) | +// | Operation (variable) | +// | Data (variable) | // +-----------------------+ -// | Operations (variable) | +// | Footer (fixed) | // +-----------------------+ // -// The "raw data" occurs immediately after the header, and the operation -// sequence occurs after the raw data. This ordering is intentional. While -// streaming an OTA, we can immediately write compressed data, but store the -// metadata in memory. At the end, we can simply append the metadata and flush -// the file. There is no need to create separate files to store the metadata -// and block data. +// The operations begin immediately after the header, and the "raw data" +// immediately follows the operation which refers to it. While streaming +// an OTA, we can immediately write the op and data, syncing after each pair, +// while storing operation metadata in memory. At the end, we compute data and +// hashes for the footer, which is placed at the tail end of the file. +// +// A missing or corrupt footer likely indicates that writing was cut off +// between writing the last operation/data pair, or the footer itself. In this +// case, the safest way to proceed is to assume the last operation is faulty. + struct CowHeader { uint64_t magic; uint16_t major_version; @@ -48,18 +54,35 @@ struct CowHeader { // Size of this struct. uint16_t header_size; - // Offset to the location of the operation sequence, and size of the - // operation sequence buffer. |ops_offset| is also the end of the - // raw data region. - uint64_t ops_offset; - uint64_t ops_size; - uint64_t num_ops; + // Size of footer struct + uint16_t footer_size; // The size of block operations, in bytes. uint32_t block_size; +} __attribute__((packed)); - // SHA256 checksums of this header, with this field set to 0. - uint8_t header_checksum[32]; +// This structure is the same size of a normal Operation, but is repurposed for the footer. +struct CowFooterOperation { + // The operation code (always kCowFooterOp). + uint8_t type; + + // If this operation reads from the data section of the COW, this contains + // the compression type of that data (see constants below). + uint8_t compression; + + // Length of Footer Data. Currently 64 for both checksums + uint16_t data_length; + + // The amount of file space used by Cow operations + uint64_t ops_size; + + // The number of cow operations in the file + uint64_t num_ops; +} __attribute__((packed)); + +struct CowFooterData { + // SHA256 checksums of Footer op + uint8_t footer_checksum[32]; // SHA256 of the operation sequence. uint8_t ops_checksum[32]; @@ -92,16 +115,31 @@ struct CowOperation { // // For zero operations (replace with all zeroes), this is unused and must // be zero. + // + // For Label operations, this is the value of the applied label. uint64_t source; } __attribute__((packed)); +static_assert(sizeof(CowOperation) == sizeof(CowFooterOperation)); + static constexpr uint8_t kCowCopyOp = 1; static constexpr uint8_t kCowReplaceOp = 2; static constexpr uint8_t kCowZeroOp = 3; +static constexpr uint8_t kCowLabelOp = 4; +static constexpr uint8_t kCowFooterOp = -1; static constexpr uint8_t kCowCompressNone = 0; static constexpr uint8_t kCowCompressGz = 1; static constexpr uint8_t kCowCompressBrotli = 2; +struct CowFooter { + CowFooterOperation op; + CowFooterData data; +} __attribute__((packed)); + +std::ostream& operator<<(std::ostream& os, CowOperation const& arg); + +int64_t GetNextOpOffset(const CowOperation& op); + } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h index 399877637..a4360aa66 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h @@ -58,6 +58,9 @@ class ICowReader { // Return the file header. virtual bool GetHeader(CowHeader* header) = 0; + // Return the file footer. + virtual bool GetFooter(CowFooter* footer) = 0; + // Return an iterator for retrieving CowOperation entries. virtual std::unique_ptr GetOpIter() = 0; @@ -89,6 +92,7 @@ class CowReader : public ICowReader { bool Parse(android::base::borrowed_fd fd); bool GetHeader(CowHeader* header) override; + bool GetFooter(CowFooter* footer) override; // Create a CowOpIter object which contains header_.num_ops // CowOperation objects. Get() returns a unique CowOperation object @@ -102,7 +106,9 @@ class CowReader : public ICowReader { android::base::unique_fd owned_fd_; android::base::borrowed_fd fd_; CowHeader header_; + CowFooter footer_; uint64_t fd_size_; + bool has_footer_; }; } // namespace snapshot diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h index 3ab377823..4cfbaaaee 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h @@ -51,6 +51,9 @@ class ICowWriter { // Encode a sequence of zeroed blocks. |size| must be a multiple of the block size. bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks); + // Add a label to the op sequence. + bool AddLabel(uint64_t label); + // Flush all pending writes. This must be called before closing the writer // to ensure that the correct headers and footers are written. virtual bool Finalize() = 0; @@ -67,6 +70,7 @@ class ICowWriter { virtual bool EmitCopy(uint64_t new_block, uint64_t old_block) = 0; virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0; virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0; + virtual bool EmitLabel(uint64_t label) = 0; bool ValidateNewBlock(uint64_t new_block); @@ -92,6 +96,7 @@ class CowWriter : public ICowWriter { virtual bool EmitCopy(uint64_t new_block, uint64_t old_block) override; virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; + virtual bool EmitLabel(uint64_t label) override; private: void SetupHeaders(); @@ -100,6 +105,7 @@ class CowWriter : public ICowWriter { bool OpenForAppend(); bool GetDataPos(uint64_t* pos); bool WriteRawData(const void* data, size_t size); + bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0); void AddOperation(const CowOperation& op); std::basic_string Compress(const void* data, size_t length); @@ -107,7 +113,9 @@ class CowWriter : public ICowWriter { android::base::unique_fd owned_fd_; android::base::borrowed_fd fd_; CowHeader header_{}; + CowFooter footer_{}; int compression_ = 0; + uint64_t next_op_pos_ = 0; // :TODO: this is not efficient, but stringstream ubsan aborts because some // bytes overflow a signed char. diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h index e293eacd0..f76f54540 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h @@ -63,6 +63,7 @@ class CompressedSnapshotWriter : public ISnapshotWriter { bool EmitCopy(uint64_t new_block, uint64_t old_block) override; bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; + bool EmitLabel(uint64_t label) override; private: android::base::unique_fd cow_device_; @@ -86,6 +87,7 @@ class OnlineKernelSnapshotWriter : public ISnapshotWriter { bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; bool EmitCopy(uint64_t new_block, uint64_t old_block) override; + bool EmitLabel(uint64_t label) override; private: android::base::unique_fd snapshot_fd_; diff --git a/fs_mgr/libsnapshot/snapshot_writer.cpp b/fs_mgr/libsnapshot/snapshot_writer.cpp index 2ede24a79..9b1ab9780 100644 --- a/fs_mgr/libsnapshot/snapshot_writer.cpp +++ b/fs_mgr/libsnapshot/snapshot_writer.cpp @@ -84,6 +84,10 @@ bool CompressedSnapshotWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t return cow_->AddZeroBlocks(new_block_start, num_blocks); } +bool CompressedSnapshotWriter::EmitLabel(uint64_t label) { + return cow_->AddLabel(label); +} + OnlineKernelSnapshotWriter::OnlineKernelSnapshotWriter(const CowOptions& options) : ISnapshotWriter(options) {} @@ -140,6 +144,11 @@ bool OnlineKernelSnapshotWriter::EmitCopy(uint64_t new_block, uint64_t old_block return EmitRawBlocks(new_block, buffer.data(), buffer.size()); } +bool OnlineKernelSnapshotWriter::EmitLabel(uint64_t) { + // Not Needed + return true; +} + std::unique_ptr OnlineKernelSnapshotWriter::OpenReader() { unique_fd fd(dup(snapshot_fd_.get())); if (fd < 0) { diff --git a/fs_mgr/libsnapshot/snapuserd.cpp b/fs_mgr/libsnapshot/snapuserd.cpp index 62ef1b063..6a82a004e 100644 --- a/fs_mgr/libsnapshot/snapuserd.cpp +++ b/fs_mgr/libsnapshot/snapuserd.cpp @@ -326,6 +326,7 @@ int Snapuserd::ReadDiskExceptions(chunk_t chunk, size_t read_size) { int Snapuserd::ReadMetadata() { reader_ = std::make_unique(); CowHeader header; + CowFooter footer; if (!reader_->Parse(cow_fd_)) { LOG(ERROR) << "Failed to parse"; @@ -337,11 +338,15 @@ int Snapuserd::ReadMetadata() { return 1; } + if (!reader_->GetFooter(&footer)) { + LOG(ERROR) << "Failed to get footer"; + return 1; + } + CHECK(header.block_size == BLOCK_SIZE); - LOG(DEBUG) << "Num-ops: " << std::hex << header.num_ops; - LOG(DEBUG) << "ops-offset: " << std::hex << header.ops_offset; - LOG(DEBUG) << "ops-size: " << std::hex << header.ops_size; + LOG(DEBUG) << "Num-ops: " << std::hex << footer.op.num_ops; + LOG(DEBUG) << "ops-size: " << std::hex << footer.op.ops_size; cowop_iter_ = reader_->GetOpIter(); @@ -373,6 +378,7 @@ int Snapuserd::ReadMetadata() { struct disk_exception* de = reinterpret_cast((char*)de_ptr.get() + offset); + if (cow_op->type == kCowFooterOp || cow_op->type == kCowLabelOp) continue; if (!(cow_op->type == kCowReplaceOp || cow_op->type == kCowZeroOp || cow_op->type == kCowCopyOp)) { LOG(ERROR) << "Unknown operation-type found: " << cow_op->type;