openkylin
/
platform_system_core
mirror of https://gitee.com/openkylin/platform_system_core.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Switch up Cow Format to be resumable 

This switches up the format to alternate ops with data, followed by a
footer containing additional meta information. This allows the file to
be resumed at arbitrary points if writing gets interrupted by power
loss.
Also adds a label op, which allows labeling future ops as connected.
If the footer is missing, Append will treat the last label as possibly
corrupt, and ignore it.

Change-Id: I126e15837d710776f9396e7afc9b0cd595e26b59
Bug: 168829493
Test: cow_api_test
This commit is contained in:
Daniel Rosenberg 2020-10-16 19:42:32 -07:00
parent b4a81ccd5a
commit 2d2fd72502
11 changed files with 433 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fs_mgr/libsnapshot/Android.bp
View File

@ -149,6 +149,7 @@ cc_defaults {
"cow_decompress.cpp",
"cow_reader.cpp",
"cow_writer.cpp",
"cow_format.cpp",
],
}

140
fs_mgr/libsnapshot/cow_api_test.cpp
View File

@ -76,13 +76,15 @@ TEST_F(CowTest, ReadWrite) {
CowReader reader;
CowHeader header;
CowFooter footer;
ASSERT_TRUE(reader.Parse(cow_->fd));
ASSERT_TRUE(reader.GetHeader(&header));
ASSERT_TRUE(reader.GetFooter(&footer));
ASSERT_EQ(header.magic, kCowMagicNumber);
ASSERT_EQ(header.major_version, kCowVersionMajor);
ASSERT_EQ(header.minor_version, kCowVersionMinor);
ASSERT_EQ(header.block_size, options.block_size);
ASSERT_EQ(header.num_ops, 4);
ASSERT_EQ(footer.op.num_ops, 4);
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
@ -105,7 +107,6 @@ TEST_F(CowTest, ReadWrite) {
ASSERT_EQ(op->compression, kCowCompressNone);
ASSERT_EQ(op->data_length, 4096);
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 106);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
@ -163,7 +164,6 @@ TEST_F(CowTest, CompressGz) {
ASSERT_EQ(op->compression, kCowCompressGz);
ASSERT_EQ(op->data_length, 56); // compressed!
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 106);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
@ -272,6 +272,7 @@ TEST_F(CowTest, GetSize) {
}
TEST_F(CowTest, Append) {
cow_->DoNotRemove();
CowOptions options;
auto writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd));
@ -325,6 +326,139 @@ TEST_F(CowTest, Append) {
ASSERT_TRUE(iter->Done());
}
TEST_F(CowTest, AppendCorrupted) {
CowOptions options;
auto writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd));
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddLabel(0));
ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer->AddLabel(1));
ASSERT_TRUE(writer->AddZeroBlocks(50, 1));
ASSERT_TRUE(writer->Finalize());
// Drop the tail end of the header. Last entry may be corrupted.
ftruncate(cow_->fd, writer->GetCowSize() - 5);
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND));
ASSERT_TRUE(writer->AddLabel(2));
ASSERT_TRUE(writer->AddZeroBlocks(50, 1));
std::string data2 = "More data!";
data2.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddLabel(3));
ASSERT_TRUE(writer->AddRawBlocks(51, data2.data(), data2.size()));
ASSERT_TRUE(writer->Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
// Read back all three operations.
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
StringSink sink;
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
auto op = &iter->Get();
ASSERT_EQ(op->type, kCowLabelOp);
ASSERT_EQ(op->source, 0);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
iter->Next();
sink.Reset();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowLabelOp);
ASSERT_EQ(op->source, 2);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowZeroOp);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowLabelOp);
ASSERT_EQ(op->source, 3);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data2);
iter->Next();
ASSERT_TRUE(iter->Done());
}
TEST_F(CowTest, AppendExtendedCorrupted) {
CowOptions options;
auto writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd));
ASSERT_TRUE(writer->AddLabel(5));
ASSERT_TRUE(writer->AddLabel(6));
std::string data = "This is some data, believe it";
data.resize(options.block_size * 2, '\0');
ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));
// fail to write the footer
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND));
ASSERT_TRUE(writer->Finalize());
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
// Read back all three operations.
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
StringSink sink;
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
auto op = &iter->Get();
ASSERT_EQ(op->type, kCowLabelOp);
ASSERT_EQ(op->source, 5);
iter->Next();
ASSERT_TRUE(iter->Done());
}
} // namespace snapshot
} // namespace android

63
fs_mgr/libsnapshot/cow_format.cpp Normal file
View File

@ -0,0 +1,63 @@
//
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <libsnapshot/cow_format.h>
#include <sys/types.h>
#include <unistd.h>
#include <android-base/logging.h>
namespace android {
namespace snapshot {
std::ostream& operator<<(std::ostream& os, CowOperation const& op) {
os << "CowOperation(type:";
if (op.type == kCowCopyOp)
os << "kCowCopyOp, ";
else if (op.type == kCowReplaceOp)
os << "kCowReplaceOp, ";
else if (op.type == kCowZeroOp)
os << "kZeroOp, ";
else if (op.type == kCowFooterOp)
os << "kCowFooterOp, ";
else if (op.type == kCowLabelOp)
os << "kCowLabelOp, ";
else
os << (int)op.type << "?,";
os << "compression:";
if (op.compression == kCowCompressNone)
os << "kCowCompressNone, ";
else if (op.compression == kCowCompressGz)
os << "kCowCompressGz, ";
else if (op.compression == kCowCompressBrotli)
os << "kCowCompressBrotli, ";
else
os << (int)op.compression << "?, ";
os << "data_length:" << op.data_length << ",\t";
os << "new_block:" << op.new_block << ",\t";
os << "source:" << op.source << ")";
return os;
}
int64_t GetNextOpOffset(const CowOperation& op) {
if (op.type == kCowReplaceOp)
return op.data_length;
else
return 0;
}
} // namespace snapshot
} // namespace android

114
fs_mgr/libsnapshot/cow_reader.cpp
View File

@ -18,17 +18,19 @@
#include <unistd.h>
#include <limits>
#include <vector>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <libsnapshot/cow_reader.h>
#include <zlib.h>
#include "cow_decompress.h"
namespace android {
namespace snapshot {
CowReader::CowReader() : fd_(-1), header_(), fd_size_(0) {}
CowReader::CowReader() : fd_(-1), header_(), footer_(), fd_size_(0), has_footer_(false) {}
static void SHA256(const void*, size_t, uint8_t[]) {
#if 0
@ -63,16 +65,6 @@ bool CowReader::Parse(android::base::borrowed_fd fd) {
return false;
}
// Validity check the ops range.
if (header_.ops_offset >= fd_size_) {
LOG(ERROR) << "ops offset " << header_.ops_offset << " larger than fd size " << fd_size_;
return false;
}
if (fd_size_ - header_.ops_offset < header_.ops_size) {
LOG(ERROR) << "ops size " << header_.ops_size << " is too large";
return false;
}
if (header_.magic != kCowMagicNumber) {
LOG(ERROR) << "Header Magic corrupted. Magic: " << header_.magic
<< "Expected: " << kCowMagicNumber;
@ -83,6 +75,11 @@ bool CowReader::Parse(android::base::borrowed_fd fd) {
<< sizeof(CowHeader);
return false;
}
if (header_.footer_size != sizeof(CowFooter)) {
LOG(ERROR) << "Footer size unknown, read " << header_.footer_size << ", expected "
<< sizeof(CowFooter);
return false;
}
if ((header_.major_version != kCowVersionMajor) ||
(header_.minor_version != kCowVersionMinor)) {
@ -94,19 +91,16 @@ bool CowReader::Parse(android::base::borrowed_fd fd) {
return false;
}
uint8_t header_csum[32];
{
CowHeader tmp = header_;
memset(&tmp.header_checksum, 0, sizeof(tmp.header_checksum));
memset(header_csum, 0, sizeof(uint8_t) * 32);
SHA256(&tmp, sizeof(tmp), header_csum);
}
if (memcmp(header_csum, header_.header_checksum, sizeof(header_csum)) != 0) {
LOG(ERROR) << "header checksum is invalid";
auto footer_pos = lseek(fd_.get(), -header_.footer_size, SEEK_END);
if (footer_pos != fd_size_ - header_.footer_size) {
LOG(ERROR) << "Failed to read full footer!";
return false;
}
if (!android::base::ReadFully(fd_, &footer_, sizeof(footer_))) {
PLOG(ERROR) << "read footer failed";
return false;
}
has_footer_ = (footer_.op.type == kCowFooterOp);
return true;
}
@ -115,74 +109,88 @@ bool CowReader::GetHeader(CowHeader* header) {
return true;
}
bool CowReader::GetFooter(CowFooter* footer) {
if (!has_footer_) return false;
*footer = footer_;
return true;
}
class CowOpIter final : public ICowOpIter {
public:
CowOpIter(std::unique_ptr<uint8_t[]>&& ops, size_t len);
CowOpIter(std::unique_ptr<std::vector<CowOperation>>&& ops);
bool Done() override;
const CowOperation& Get() override;
void Next() override;
private:
bool HasNext();
std::unique_ptr<uint8_t[]> ops_;
const uint8_t* pos_;
const uint8_t* end_;
bool done_;
std::unique_ptr<std::vector<CowOperation>> ops_;
std::vector<CowOperation>::iterator op_iter_;
};
CowOpIter::CowOpIter(std::unique_ptr<uint8_t[]>&& ops, size_t len)
: ops_(std::move(ops)), pos_(ops_.get()), end_(pos_ + len), done_(!HasNext()) {}
bool CowOpIter::Done() {
return done_;
CowOpIter::CowOpIter(std::unique_ptr<std::vector<CowOperation>>&& ops) {
ops_ = std::move(ops);
op_iter_ = ops_.get()->begin();
}
bool CowOpIter::HasNext() {
return pos_ < end_ && size_t(end_ - pos_) >= sizeof(CowOperation);
bool CowOpIter::Done() {
return op_iter_ == ops_.get()->end();
}
void CowOpIter::Next() {
CHECK(!Done());
pos_ += sizeof(CowOperation);
if (!HasNext()) done_ = true;
op_iter_++;
}
const CowOperation& CowOpIter::Get() {
CHECK(!Done());
CHECK(HasNext());
return *reinterpret_cast<const CowOperation*>(pos_);
return (*op_iter_);
}
std::unique_ptr<ICowOpIter> CowReader::GetOpIter() {
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
uint64_t pos = lseek(fd_.get(), sizeof(header_), SEEK_SET);
if (pos != sizeof(header_)) {
PLOG(ERROR) << "lseek ops failed";
return nullptr;
}
auto ops_buffer = std::make_unique<uint8_t[]>(header_.ops_size);
if (!android::base::ReadFully(fd_, ops_buffer.get(), header_.ops_size)) {
PLOG(ERROR) << "read ops failed";
return nullptr;
auto ops_buffer = std::make_unique<std::vector<CowOperation>>();
if (has_footer_) ops_buffer->reserve(footer_.op.num_ops);
uint64_t current_op_num = 0;
uint64_t last_pos = fd_size_ - (has_footer_ ? sizeof(footer_) : sizeof(CowOperation));
// Alternating op and data
while (pos < last_pos) {
ops_buffer->resize(current_op_num + 1);
if (!android::base::ReadFully(fd_, ops_buffer->data() + current_op_num,
sizeof(CowOperation))) {
PLOG(ERROR) << "read op failed";
return nullptr;
}
auto current_op = ops_buffer->data()[current_op_num];
pos = lseek(fd_.get(), GetNextOpOffset(current_op), SEEK_CUR);
current_op_num++;
}
uint8_t csum[32];
memset(csum, 0, sizeof(uint8_t) * 32);
SHA256(ops_buffer.get(), header_.ops_size, csum);
if (memcmp(csum, header_.ops_checksum, sizeof(csum)) != 0) {
LOG(ERROR) << "ops checksum does not match";
return nullptr;
if (has_footer_) {
SHA256(ops_buffer.get()->data(), footer_.op.ops_size, csum);
if (memcmp(csum, footer_.data.ops_checksum, sizeof(csum)) != 0) {
LOG(ERROR) << "ops checksum does not match";
return nullptr;
}
} else {
LOG(INFO) << "No Footer, recovered data";
}
return std::make_unique<CowOpIter>(std::move(ops_buffer), header_.ops_size);
return std::make_unique<CowOpIter>(std::move(ops_buffer));
}
bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {
// Validate the offset, taking care to acknowledge possible overflow of offset+len.
if (offset < sizeof(header_) || offset >= header_.ops_offset || len >= fd_size_ ||
offset + len > header_.ops_offset) {
if (offset < sizeof(header_) || offset >= fd_size_ - sizeof(footer_) || len >= fd_size_ ||
offset + len > fd_size_ - sizeof(footer_)) {
LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes";
return false;
}

133
fs_mgr/libsnapshot/cow_writer.cpp
View File

@ -18,6 +18,7 @@
#include <unistd.h>
#include <limits>
#include <queue>
#include <android-base/file.h>
#include <android-base/logging.h>
@ -65,6 +66,10 @@ bool ICowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
return EmitZeroBlocks(new_block_start, num_blocks);
}
bool ICowWriter::AddLabel(uint64_t label) {
return EmitLabel(label);
}
bool ICowWriter::ValidateNewBlock(uint64_t new_block) {
if (options_.max_blocks && new_block >= options_.max_blocks.value()) {
LOG(ERROR) << "New block " << new_block << " exceeds maximum block count "
@ -84,7 +89,11 @@ void CowWriter::SetupHeaders() {
header_.major_version = kCowVersionMajor;
header_.minor_version = kCowVersionMinor;
header_.header_size = sizeof(CowHeader);
header_.footer_size = sizeof(CowFooter);
header_.block_size = options_.block_size;
footer_ = {};
footer_.op.data_length = 64;
footer_.op.type = kCowFooterOp;
}
bool CowWriter::ParseOptions() {
@ -143,33 +152,53 @@ bool CowWriter::OpenForWrite() {
return false;
}
header_.ops_offset = header_.header_size;
next_op_pos_ = sizeof(header_);
return true;
}
bool CowWriter::OpenForAppend() {
auto reader = std::make_unique<CowReader>();
bool incomplete = false;
std::queue<CowOperation> toAdd;
if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) {
return false;
}
incomplete = !reader->GetFooter(&footer_);
options_.block_size = header_.block_size;
// Reset this, since we're going to reimport all operations.
header_.num_ops = 0;
footer_.op.num_ops = 0;
next_op_pos_ = sizeof(header_);
auto iter = reader->GetOpIter();
while (!iter->Done()) {
auto& op = iter->Get();
AddOperation(op);
CowOperation op = iter->Get();
if (op.type == kCowFooterOp) break;
if (incomplete) {
// Last operation translation may be corrupt. Wait to add it.
if (op.type == kCowLabelOp) {
while (!toAdd.empty()) {
AddOperation(toAdd.front());
toAdd.pop();
}
}
toAdd.push(op);
} else {
AddOperation(op);
}
iter->Next();
}
// Free reader so we own the descriptor position again.
reader = nullptr;
// Seek to the end of the data section.
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
// Position for new writing
if (ftruncate(fd_.get(), next_op_pos_) != 0) {
PLOG(ERROR) << "Failed to trim file";
return false;
}
if (lseek(fd_.get(), 0, SEEK_END) < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
@ -181,22 +210,18 @@ bool CowWriter::EmitCopy(uint64_t new_block, uint64_t old_block) {
op.type = kCowCopyOp;
op.new_block = new_block;
op.source = old_block;
AddOperation(op);
return true;
return WriteOperation(op);
}
bool CowWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) {
uint64_t pos;
if (!GetDataPos(&pos)) {
return false;
}
const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
uint64_t pos;
for (size_t i = 0; i < size / header_.block_size; i++) {
CowOperation op = {};
op.type = kCowReplaceOp;
op.new_block = new_block_start + i;
op.source = pos;
GetDataPos(&pos);
op.source = pos + sizeof(op);
if (compression_) {
auto data = Compress(iter, header_.block_size);
@ -208,26 +233,23 @@ bool CowWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t
LOG(ERROR) << "Compressed block is too large: " << data.size() << " bytes";
return false;
}
if (!WriteRawData(data.data(), data.size())) {
op.compression = compression_;
op.data_length = static_cast<uint16_t>(data.size());
if (!WriteOperation(op, data.data(), data.size())) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
op.compression = compression_;
op.data_length = static_cast<uint16_t>(data.size());
pos += data.size();
} else {
op.data_length = static_cast<uint16_t>(header_.block_size);
pos += header_.block_size;
if (!WriteOperation(op, iter, header_.block_size)) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
}
AddOperation(op);
iter += header_.block_size;
}
if (!compression_ && !WriteRawData(data, size)) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
return true;
}
@ -237,11 +259,18 @@ bool CowWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
op.type = kCowZeroOp;
op.new_block = new_block_start + i;
op.source = 0;
AddOperation(op);
WriteOperation(op);
}
return true;
}
bool CowWriter::EmitLabel(uint64_t label) {
CowOperation op = {};
op.type = kCowLabelOp;
op.source = label;
return WriteOperation(op);
}
std::basic_string<uint8_t> CowWriter::Compress(const void* data, size_t length) {
switch (compression_) {
case kCowCompressGz: {
@ -294,33 +323,27 @@ static void SHA256(const void*, size_t, uint8_t[]) {
}
bool CowWriter::Finalize() {
header_.ops_size = ops_.size();
footer_.op.ops_size = ops_.size() + sizeof(footer_.op);
uint64_t pos;
memset(header_.ops_checksum, 0, sizeof(uint8_t) * 32);
memset(header_.header_checksum, 0, sizeof(uint8_t) * 32);
SHA256(ops_.data(), ops_.size(), header_.ops_checksum);
SHA256(&header_, sizeof(header_), header_.header_checksum);
if (lseek(fd_.get(), 0, SEEK_SET)) {
PLOG(ERROR) << "lseek failed";
if (!GetDataPos(&pos)) {
PLOG(ERROR) << "failed to get file position";
return false;
}
if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "write header failed";
return false;
}
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek ops failed";
return false;
}
if (!WriteFully(fd_, ops_.data(), ops_.size())) {
PLOG(ERROR) << "write ops failed";
memset(&footer_.data.ops_checksum, 0, sizeof(uint8_t) * 32);
memset(&footer_.data.footer_checksum, 0, sizeof(uint8_t) * 32);
SHA256(ops_.data(), ops_.size(), footer_.data.ops_checksum);
SHA256(&footer_.op, sizeof(footer_.op), footer_.data.footer_checksum);
// Write out footer at end of file
if (!android::base::WriteFully(fd_, reinterpret_cast<const uint8_t*>(&footer_),
sizeof(footer_))) {
PLOG(ERROR) << "write footer failed";
return false;
}
// Re-position for any subsequent writes.
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
if (lseek(fd_.get(), pos, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek ops failed";
return false;
}
@ -328,7 +351,7 @@ bool CowWriter::Finalize() {
}
uint64_t CowWriter::GetCowSize() {
return header_.ops_offset + header_.num_ops * sizeof(CowOperation);
return next_op_pos_ + sizeof(footer_);
}
bool CowWriter::GetDataPos(uint64_t* pos) {
@ -341,8 +364,19 @@ bool CowWriter::GetDataPos(uint64_t* pos) {
return true;
}
bool CowWriter::WriteOperation(const CowOperation& op, const void* data, size_t size) {
if (!android::base::WriteFully(fd_, reinterpret_cast<const uint8_t*>(&op), sizeof(op))) {
return false;
}
if (data != NULL && size > 0)
if (!WriteRawData(data, size)) return false;
AddOperation(op);
return !fsync(fd_.get());
}
void CowWriter::AddOperation(const CowOperation& op) {
header_.num_ops++;
footer_.op.num_ops++;
next_op_pos_ += sizeof(CowOperation) + GetNextOpOffset(op);
ops_.insert(ops_.size(), reinterpret_cast<const uint8_t*>(&op), sizeof(op));
}
@ -350,7 +384,6 @@ bool CowWriter::WriteRawData(const void* data, size_t size) {
if (!android::base::WriteFully(fd_, data, size)) {
return false;
}
header_.ops_offset += size;
return true;
}

70
fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
View File

@ -15,6 +15,7 @@
#pragma once
#include <stdint.h>
#include <string>
namespace android {
namespace snapshot {
@ -29,17 +30,22 @@ static constexpr uint32_t kCowVersionMinor = 0;
// +-----------------------+
// | Header (fixed) |
// +-----------------------+
// | Raw Data (variable) |
// | Operation (variable) |
// | Data (variable) |
// +-----------------------+
// | Operations (variable) |
// | Footer (fixed) |
// +-----------------------+
//
// The "raw data" occurs immediately after the header, and the operation
// sequence occurs after the raw data. This ordering is intentional. While
// streaming an OTA, we can immediately write compressed data, but store the
// metadata in memory. At the end, we can simply append the metadata and flush
// the file. There is no need to create separate files to store the metadata
// and block data.
// The operations begin immediately after the header, and the "raw data"
// immediately follows the operation which refers to it. While streaming
// an OTA, we can immediately write the op and data, syncing after each pair,
// while storing operation metadata in memory. At the end, we compute data and
// hashes for the footer, which is placed at the tail end of the file.
//
// A missing or corrupt footer likely indicates that writing was cut off
// between writing the last operation/data pair, or the footer itself. In this
// case, the safest way to proceed is to assume the last operation is faulty.
struct CowHeader {
uint64_t magic;
uint16_t major_version;
@ -48,18 +54,35 @@ struct CowHeader {
// Size of this struct.
uint16_t header_size;
// Offset to the location of the operation sequence, and size of the
// operation sequence buffer. |ops_offset| is also the end of the
// raw data region.
uint64_t ops_offset;
uint64_t ops_size;
uint64_t num_ops;
// Size of footer struct
uint16_t footer_size;
// The size of block operations, in bytes.
uint32_t block_size;
} __attribute__((packed));
// SHA256 checksums of this header, with this field set to 0.
uint8_t header_checksum[32];
// This structure is the same size of a normal Operation, but is repurposed for the footer.
struct CowFooterOperation {
// The operation code (always kCowFooterOp).
uint8_t type;
// If this operation reads from the data section of the COW, this contains
// the compression type of that data (see constants below).
uint8_t compression;
// Length of Footer Data. Currently 64 for both checksums
uint16_t data_length;
// The amount of file space used by Cow operations
uint64_t ops_size;
// The number of cow operations in the file
uint64_t num_ops;
} __attribute__((packed));
struct CowFooterData {
// SHA256 checksums of Footer op
uint8_t footer_checksum[32];
// SHA256 of the operation sequence.
uint8_t ops_checksum[32];
@ -92,16 +115,31 @@ struct CowOperation {
//
// For zero operations (replace with all zeroes), this is unused and must
// be zero.
//
// For Label operations, this is the value of the applied label.
uint64_t source;
} __attribute__((packed));
static_assert(sizeof(CowOperation) == sizeof(CowFooterOperation));
static constexpr uint8_t kCowCopyOp = 1;
static constexpr uint8_t kCowReplaceOp = 2;
static constexpr uint8_t kCowZeroOp = 3;
static constexpr uint8_t kCowLabelOp = 4;
static constexpr uint8_t kCowFooterOp = -1;
static constexpr uint8_t kCowCompressNone = 0;
static constexpr uint8_t kCowCompressGz = 1;
static constexpr uint8_t kCowCompressBrotli = 2;
struct CowFooter {
CowFooterOperation op;
CowFooterData data;
} __attribute__((packed));
std::ostream& operator<<(std::ostream& os, CowOperation const& arg);
int64_t GetNextOpOffset(const CowOperation& op);
} // namespace snapshot
} // namespace android

6
fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
View File

@ -58,6 +58,9 @@ class ICowReader {
// Return the file header.
virtual bool GetHeader(CowHeader* header) = 0;
// Return the file footer.
virtual bool GetFooter(CowFooter* footer) = 0;
// Return an iterator for retrieving CowOperation entries.
virtual std::unique_ptr<ICowOpIter> GetOpIter() = 0;
@ -89,6 +92,7 @@ class CowReader : public ICowReader {
bool Parse(android::base::borrowed_fd fd);
bool GetHeader(CowHeader* header) override;
bool GetFooter(CowFooter* footer) override;
// Create a CowOpIter object which contains header_.num_ops
// CowOperation objects. Get() returns a unique CowOperation object
@ -102,7 +106,9 @@ class CowReader : public ICowReader {
android::base::unique_fd owned_fd_;
android::base::borrowed_fd fd_;
CowHeader header_;
CowFooter footer_;
uint64_t fd_size_;
bool has_footer_;
};
} // namespace snapshot

8
fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
View File

@ -51,6 +51,9 @@ class ICowWriter {
// Encode a sequence of zeroed blocks. |size| must be a multiple of the block size.
bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks);
// Add a label to the op sequence.
bool AddLabel(uint64_t label);
// Flush all pending writes. This must be called before closing the writer
// to ensure that the correct headers and footers are written.
virtual bool Finalize() = 0;
@ -67,6 +70,7 @@ class ICowWriter {
virtual bool EmitCopy(uint64_t new_block, uint64_t old_block) = 0;
virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0;
virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;
virtual bool EmitLabel(uint64_t label) = 0;
bool ValidateNewBlock(uint64_t new_block);
@ -92,6 +96,7 @@ class CowWriter : public ICowWriter {
virtual bool EmitCopy(uint64_t new_block, uint64_t old_block) override;
virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
virtual bool EmitLabel(uint64_t label) override;
private:
void SetupHeaders();
@ -100,6 +105,7 @@ class CowWriter : public ICowWriter {
bool OpenForAppend();
bool GetDataPos(uint64_t* pos);
bool WriteRawData(const void* data, size_t size);
bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0);
void AddOperation(const CowOperation& op);
std::basic_string<uint8_t> Compress(const void* data, size_t length);
@ -107,7 +113,9 @@ class CowWriter : public ICowWriter {
android::base::unique_fd owned_fd_;
android::base::borrowed_fd fd_;
CowHeader header_{};
CowFooter footer_{};
int compression_ = 0;
uint64_t next_op_pos_ = 0;
// :TODO: this is not efficient, but stringstream ubsan aborts because some
// bytes overflow a signed char.

2
fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h
View File

@ -63,6 +63,7 @@ class CompressedSnapshotWriter : public ISnapshotWriter {
bool EmitCopy(uint64_t new_block, uint64_t old_block) override;
bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
bool EmitLabel(uint64_t label) override;
private:
android::base::unique_fd cow_device_;
@ -86,6 +87,7 @@ class OnlineKernelSnapshotWriter : public ISnapshotWriter {
bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
bool EmitCopy(uint64_t new_block, uint64_t old_block) override;
bool EmitLabel(uint64_t label) override;
private:
android::base::unique_fd snapshot_fd_;

9
fs_mgr/libsnapshot/snapshot_writer.cpp
View File

@ -84,6 +84,10 @@ bool CompressedSnapshotWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t
return cow_->AddZeroBlocks(new_block_start, num_blocks);
}
bool CompressedSnapshotWriter::EmitLabel(uint64_t label) {
return cow_->AddLabel(label);
}
OnlineKernelSnapshotWriter::OnlineKernelSnapshotWriter(const CowOptions& options)
: ISnapshotWriter(options) {}
@ -140,6 +144,11 @@ bool OnlineKernelSnapshotWriter::EmitCopy(uint64_t new_block, uint64_t old_block
return EmitRawBlocks(new_block, buffer.data(), buffer.size());
}
bool OnlineKernelSnapshotWriter::EmitLabel(uint64_t) {
// Not Needed
return true;
}
std::unique_ptr<FileDescriptor> OnlineKernelSnapshotWriter::OpenReader() {
unique_fd fd(dup(snapshot_fd_.get()));
if (fd < 0) {

12
fs_mgr/libsnapshot/snapuserd.cpp
View File

@ -326,6 +326,7 @@ int Snapuserd::ReadDiskExceptions(chunk_t chunk, size_t read_size) {
int Snapuserd::ReadMetadata() {
reader_ = std::make_unique<CowReader>();
CowHeader header;
CowFooter footer;
if (!reader_->Parse(cow_fd_)) {
LOG(ERROR) << "Failed to parse";
@ -337,11 +338,15 @@ int Snapuserd::ReadMetadata() {
return 1;
}
if (!reader_->GetFooter(&footer)) {
LOG(ERROR) << "Failed to get footer";
return 1;
}
CHECK(header.block_size == BLOCK_SIZE);
LOG(DEBUG) << "Num-ops: " << std::hex << header.num_ops;
LOG(DEBUG) << "ops-offset: " << std::hex << header.ops_offset;
LOG(DEBUG) << "ops-size: " << std::hex << header.ops_size;
LOG(DEBUG) << "Num-ops: " << std::hex << footer.op.num_ops;
LOG(DEBUG) << "ops-size: " << std::hex << footer.op.ops_size;
cowop_iter_ = reader_->GetOpIter();
@ -373,6 +378,7 @@ int Snapuserd::ReadMetadata() {
struct disk_exception* de =
reinterpret_cast<struct disk_exception*>((char*)de_ptr.get() + offset);
if (cow_op->type == kCowFooterOp || cow_op->type == kCowLabelOp) continue;
if (!(cow_op->type == kCowReplaceOp || cow_op->type == kCowZeroOp ||
cow_op->type == kCowCopyOp)) {
LOG(ERROR) << "Unknown operation-type found: " << cow_op->type;