openkylin
/
platform_system_core
mirror of https://gitee.com/openkylin/platform_system_core.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

libsnapshot: Add an append mode to CowWriter. 

When in append mode, CowWriter will re-open the existing COW and resume
writing at the end of the old data position. All existing operations
will be reimported and buffered in memory.

The size calculation has been simplified to make this work. We now
advance ops_offset and no longer track the number of bytes written.

Additionally, a "header_size" field has been added to the header. This
was missing from the original format and is useful for introducing
forward compatibility later.

Finally, Finalize has been renamed to Flush. It's still mandatory, but
it can be called multiple times to continue appending data without
reopening.

Bug: 168554689
Test: cow_api_test gtest
Change-Id: I637e99ae08a4db5b273c06318e6db523ea8ec7c5
This commit is contained in:
David Anderson 2020-09-14 17:05:04 -07:00
parent a652877bd6
commit a889c87b0f
9 changed files with 178 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fs_mgr/libsnapshot/Android.bp
View File

@ -412,6 +412,7 @@ cc_test {
test_min_api_level: 30,
auto_gen_config: true,
require_root: false,
host_supported: true,
}
cc_binary {

68
fs_mgr/libsnapshot/cow_api_test.cpp
View File

@ -70,7 +70,7 @@ TEST_F(CowTest, ReadWrite) {
ASSERT_TRUE(writer.AddCopy(10, 20));
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.AddZeroBlocks(51, 2));
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -105,7 +105,7 @@ TEST_F(CowTest, ReadWrite) {
ASSERT_EQ(op->compression, kCowCompressNone);
ASSERT_EQ(op->data_length, 4096);
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 104);
ASSERT_EQ(op->source, 106);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
@ -145,7 +145,7 @@ TEST_F(CowTest, CompressGz) {
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -163,7 +163,7 @@ TEST_F(CowTest, CompressGz) {
ASSERT_EQ(op->compression, kCowCompressGz);
ASSERT_EQ(op->data_length, 56); // compressed!
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 104);
ASSERT_EQ(op->source, 106);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
@ -182,7 +182,7 @@ TEST_F(CowTest, CompressTwoBlocks) {
data.resize(options.block_size * 2, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -224,7 +224,7 @@ TEST_P(CompressionTest, HorribleSink) {
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -259,7 +259,7 @@ TEST_F(CowTest, GetSize) {
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.AddZeroBlocks(51, 2));
auto size_before = writer.GetCowSize();
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
auto size_after = writer.GetCowSize();
ASSERT_EQ(size_before, size_after);
struct stat buf;
@ -271,6 +271,60 @@ TEST_F(CowTest, GetSize) {
ASSERT_EQ(buf.st_size, writer.GetCowSize());
}
TEST_F(CowTest, Append) {
CowOptions options;
auto writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd));
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer->Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND));
std::string data2 = "More data!";
data2.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddRawBlocks(51, data2.data(), data2.size()));
ASSERT_TRUE(writer->Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
// Read back both operations.
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
StringSink sink;
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
auto op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
iter->Next();
sink.Reset();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data2);
iter->Next();
ASSERT_TRUE(iter->Done());
}
} // namespace snapshot
} // namespace android

5
fs_mgr/libsnapshot/cow_reader.cpp
View File

@ -78,6 +78,11 @@ bool CowReader::Parse(android::base::borrowed_fd fd) {
<< "Expected: " << kCowMagicNumber;
return false;
}
if (header_.header_size != sizeof(CowHeader)) {
LOG(ERROR) << "Header size unknown, read " << header_.header_size << ", expected "
<< sizeof(CowHeader);
return false;
}
if ((header_.major_version != kCowVersionMajor) ||
(header_.minor_version != kCowVersionMinor)) {

2
fs_mgr/libsnapshot/cow_snapuserd_test.cpp
View File

@ -128,7 +128,7 @@ TEST_F(SnapuserdTest, ReadWrite) {
ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_2.get(), size));
// Flush operations
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);

138
fs_mgr/libsnapshot/cow_writer.cpp
View File

@ -23,6 +23,7 @@
#include <android-base/logging.h>
#include <android-base/unique_fd.h>
#include <brotli/encode.h>
#include <libsnapshot/cow_reader.h>
#include <libsnapshot/cow_writer.h>
#include <zlib.h>
@ -40,17 +41,48 @@ void CowWriter::SetupHeaders() {
header_.magic = kCowMagicNumber;
header_.major_version = kCowVersionMajor;
header_.minor_version = kCowVersionMinor;
header_.header_size = sizeof(CowHeader);
header_.block_size = options_.block_size;
}
bool CowWriter::Initialize(android::base::unique_fd&& fd) {
owned_fd_ = std::move(fd);
return Initialize(android::base::borrowed_fd{owned_fd_});
bool CowWriter::ParseOptions() {
if (options_.compression == "gz") {
compression_ = kCowCompressGz;
} else if (options_.compression == "brotli") {
compression_ = kCowCompressBrotli;
} else if (options_.compression == "none") {
compression_ = kCowCompressNone;
} else if (!options_.compression.empty()) {
LOG(ERROR) << "unrecognized compression: " << options_.compression;
return false;
}
return true;
}
bool CowWriter::Initialize(android::base::borrowed_fd fd) {
bool CowWriter::Initialize(android::base::unique_fd&& fd, OpenMode mode) {
owned_fd_ = std::move(fd);
return Initialize(android::base::borrowed_fd{owned_fd_}, mode);
}
bool CowWriter::Initialize(android::base::borrowed_fd fd, OpenMode mode) {
fd_ = fd;
if (!ParseOptions()) {
return false;
}
switch (mode) {
case OpenMode::WRITE:
return OpenForWrite();
case OpenMode::APPEND:
return OpenForAppend();
default:
LOG(ERROR) << "Unknown open mode in CowWriter";
return false;
}
}
bool CowWriter::OpenForWrite() {
// This limitation is tied to the data field size in CowOperation.
if (header_.block_size > std::numeric_limits<uint16_t>::max()) {
LOG(ERROR) << "Block size is too large";
@ -62,35 +94,52 @@ bool CowWriter::Initialize(android::base::borrowed_fd fd) {
return false;
}
if (options_.compression == "gz") {
compression_ = kCowCompressGz;
} else if (options_.compression == "brotli") {
compression_ = kCowCompressBrotli;
} else if (options_.compression == "none") {
compression_ = kCowCompressNone;
} else if (!options_.compression.empty()) {
LOG(ERROR) << "unrecognized compression: " << options_.compression;
// Headers are not complete, but this ensures the file is at the right
// position.
if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "write failed";
return false;
}
// Headers are not complete, but this ensures the file is at the right
// position.
if (!WriteFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "write failed";
header_.ops_offset = header_.header_size;
return true;
}
bool CowWriter::OpenForAppend() {
auto reader = std::make_unique<CowReader>();
if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) {
return false;
}
options_.block_size = header_.block_size;
// Reset this, since we're going to reimport all operations.
header_.num_ops = 0;
auto iter = reader->GetOpIter();
while (!iter->Done()) {
auto& op = iter->Get();
AddOperation(op);
iter->Next();
}
// Free reader so we own the descriptor position again.
reader = nullptr;
// Seek to the end of the data section.
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
return true;
}
bool CowWriter::AddCopy(uint64_t new_block, uint64_t old_block) {
header_.num_ops++;
CowOperation op = {};
op.type = kCowCopyOp;
op.new_block = new_block;
op.source = old_block;
ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));
AddOperation(op);
return true;
}
@ -108,8 +157,6 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t
const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
for (size_t i = 0; i < size / header_.block_size; i++) {
header_.num_ops++;
CowOperation op = {};
op.type = kCowReplaceOp;
op.new_block = new_block_start + i;
@ -125,7 +172,7 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t
LOG(ERROR) << "Compressed block is too large: " << data.size() << " bytes";
return false;
}
if (!WriteFully(fd_, data.data(), data.size())) {
if (!WriteRawData(data.data(), data.size())) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
@ -137,11 +184,11 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t
pos += header_.block_size;
}
ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));
AddOperation(op);
iter += header_.block_size;
}
if (!compression_ && !WriteFully(fd_, data, size)) {
if (!compression_ && !WriteRawData(data, size)) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
@ -150,13 +197,11 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t
bool CowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
for (uint64_t i = 0; i < num_blocks; i++) {
header_.num_ops++;
CowOperation op = {};
op.type = kCowZeroOp;
op.new_block = new_block_start + i;
op.source = 0;
ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));
AddOperation(op);
}
return true;
}
@ -212,17 +257,7 @@ static void SHA256(const void*, size_t, uint8_t[]) {
#endif
}
bool CowWriter::Finalize() {
// If both fields are set then Finalize is already called.
if (header_.ops_offset > 0 && header_.ops_size > 0) {
return true;
}
auto offs = lseek(fd_.get(), 0, SEEK_CUR);
if (offs < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
header_.ops_offset = offs;
bool CowWriter::Flush() {
header_.ops_size = ops_.size();
memset(header_.ops_checksum, 0, sizeof(uint8_t) * 32);
@ -235,8 +270,6 @@ bool CowWriter::Finalize() {
PLOG(ERROR) << "lseek failed";
return false;
}
// Header is already written, calling WriteFully will increment
// bytes_written_. So use android::base::WriteFully() here.
if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "write header failed";
return false;
@ -250,13 +283,16 @@ bool CowWriter::Finalize() {
return false;
}
// clear ops_ so that subsequent calls to GetSize() still works.
ops_.clear();
// Re-position for any subsequent writes.
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek ops failed";
return false;
}
return true;
}
size_t CowWriter::GetCowSize() {
return bytes_written_ + ops_.size() * sizeof(ops_[0]);
return header_.ops_offset + header_.num_ops * sizeof(CowOperation);
}
bool CowWriter::GetDataPos(uint64_t* pos) {
@ -269,9 +305,17 @@ bool CowWriter::GetDataPos(uint64_t* pos) {
return true;
}
bool CowWriter::WriteFully(base::borrowed_fd fd, const void* data, size_t size) {
bytes_written_ += size;
return android::base::WriteFully(fd, data, size);
void CowWriter::AddOperation(const CowOperation& op) {
header_.num_ops++;
ops_.insert(ops_.size(), reinterpret_cast<const uint8_t*>(&op), sizeof(op));
}
bool CowWriter::WriteRawData(const void* data, size_t size) {
if (!android::base::WriteFully(fd_, data, size)) {
return false;
}
header_.ops_offset += size;
return true;
}
} // namespace snapshot

2
fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp
View File

@ -375,7 +375,7 @@ bool NonAbEstimator::AnalyzePartition(const std::string& partition_name) {
}
}
if (!writer->Finalize()) {
if (!writer->Flush()) {
return false;
}

3
fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
View File

@ -45,6 +45,9 @@ struct CowHeader {
uint16_t major_version;
uint16_t minor_version;
// Size of this struct.
uint16_t header_size;
// Offset to the location of the operation sequence, and size of the
// operation sequence buffer. |ops_offset| is also the end of the
// raw data region.

24
fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
View File

@ -47,12 +47,11 @@ class ICowWriter {
// Encode a sequence of zeroed blocks. |size| must be a multiple of the block size.
virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;
// Finalize all COW operations and flush pending writes.
// Return true if successful.
virtual bool Finalize() = 0;
// Flush all pending writes. This must be called before closing the writer
// to ensure that the correct headers and footers are written.
virtual bool Flush() = 0;
// Return 0 if failed, on success return number of bytes the cow image would be
// after calling Finalize();
// Return number of bytes the cow image occupies on disk.
virtual size_t GetCowSize() = 0;
protected:
@ -61,24 +60,30 @@ class ICowWriter {
class CowWriter : public ICowWriter {
public:
enum class OpenMode { WRITE, APPEND };
explicit CowWriter(const CowOptions& options);
// Set up the writer.
bool Initialize(android::base::unique_fd&& fd);
bool Initialize(android::base::borrowed_fd fd);
bool Initialize(android::base::unique_fd&& fd, OpenMode mode = OpenMode::WRITE);
bool Initialize(android::base::borrowed_fd fd, OpenMode mode = OpenMode::WRITE);
bool AddCopy(uint64_t new_block, uint64_t old_block) override;
bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
bool Finalize() override;
bool Flush() override;
size_t GetCowSize() override;
private:
void SetupHeaders();
bool ParseOptions();
bool OpenForWrite();
bool OpenForAppend();
bool GetDataPos(uint64_t* pos);
bool WriteFully(base::borrowed_fd fd, const void* data, size_t size);
bool WriteRawData(const void* data, size_t size);
void AddOperation(const CowOperation& op);
std::basic_string<uint8_t> Compress(const void* data, size_t length);
private:
@ -90,7 +95,6 @@ class CowWriter : public ICowWriter {
// :TODO: this is not efficient, but stringstream ubsan aborts because some
// bytes overflow a signed char.
std::basic_string<uint8_t> ops_;
std::atomic<size_t> bytes_written_ = 0;
};
} // namespace snapshot

2
fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp
View File

@ -204,7 +204,7 @@ bool PayloadConverter::ProcessPartition(const PartitionUpdate& update) {
}
}
if (!writer_->Finalize()) {
if (!writer_->Flush()) {
LOG(ERROR) << "Unable to finalize COW for " << partition_name;
return false;
}