From f344d632222c068dfd897059f9f0268413c60fd9 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Sat, 2 Mar 2019 00:28:35 -0800 Subject: [PATCH] Support FiemapWriters that extend across multiple files. This introduces a new SplitFiemap class that will divide an allocation request across multiple FiemapWriters. This is primarily useful on filesystems that have onerous restrictions on maximum file sizes. Vfat, for example, supports a maximum of 4GiB, which is too small to satisfy larger userdata size requests. Bug: 126230649 Test: fiemap_writer_test gtest Change-Id: I3c95d341e4e94e0c44bbf0e8553c34ccfdcd155b --- fs_mgr/libfiemap_writer/Android.bp | 6 + fs_mgr/libfiemap_writer/fiemap_writer.cpp | 4 +- .../libfiemap_writer/fiemap_writer_test.cpp | 68 +++++- .../libfiemap_writer/split_fiemap_writer.h | 79 +++++++ .../libfiemap_writer/split_fiemap_writer.cpp | 214 ++++++++++++++++++ fs_mgr/libfiemap_writer/utility.cpp | 66 ++++++ fs_mgr/libfiemap_writer/utility.h | 30 +++ 7 files changed, 464 insertions(+), 3 deletions(-) create mode 100644 fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h create mode 100644 fs_mgr/libfiemap_writer/split_fiemap_writer.cpp create mode 100644 fs_mgr/libfiemap_writer/utility.cpp create mode 100644 fs_mgr/libfiemap_writer/utility.h diff --git a/fs_mgr/libfiemap_writer/Android.bp b/fs_mgr/libfiemap_writer/Android.bp index f4d03eccb..746381050 100644 --- a/fs_mgr/libfiemap_writer/Android.bp +++ b/fs_mgr/libfiemap_writer/Android.bp @@ -25,6 +25,12 @@ cc_library_static { srcs: [ "fiemap_writer.cpp", + "split_fiemap_writer.cpp", + "utility.cpp", + ], + + static_libs: [ + "libext4_utils", ], header_libs: [ diff --git a/fs_mgr/libfiemap_writer/fiemap_writer.cpp b/fs_mgr/libfiemap_writer/fiemap_writer.cpp index 9aa56e170..3d418764b 100644 --- a/fs_mgr/libfiemap_writer/fiemap_writer.cpp +++ b/fs_mgr/libfiemap_writer/fiemap_writer.cpp @@ -624,8 +624,8 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s fmap->fs_type_ = fs_type; fmap->block_size_ = blocksz; - LOG(INFO) << "Successfully created FiemapWriter for file " << abs_path << " on block device " - << bdev_path; + LOG(VERBOSE) << "Successfully created FiemapWriter for file " << abs_path << " on block device " + << bdev_path; return fmap; } diff --git a/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp b/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp index e01a0fb61..ab4efae31 100644 --- a/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp +++ b/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp @@ -33,8 +33,10 @@ #include #include #include - #include +#include + +#include "utility.h" using namespace std; using namespace std::string_literals; @@ -59,6 +61,24 @@ class FiemapWriterTest : public ::testing::Test { std::string testfile; }; +class SplitFiemapTest : public ::testing::Test { + protected: + void SetUp() override { + const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info(); + testfile = gTestDir + "/"s + tinfo->name(); + } + + void TearDown() override { + std::string message; + if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) { + cerr << "Could not remove all split files: " << message; + } + } + + // name of the file we use for testing + std::string testfile; +}; + TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) { // Try creating a file of size ~100TB but aligned to // 512 byte to make sure block alignment tests don't @@ -168,6 +188,52 @@ TEST_F(FiemapWriterTest, FileDeletedOnError) { EXPECT_EQ(errno, ENOENT); } +TEST_F(FiemapWriterTest, MaxBlockSize) { + ASSERT_GT(DetermineMaximumFileSize(testfile), 0); +} + +TEST_F(SplitFiemapTest, Create) { + auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32); + ASSERT_NE(ptr, nullptr); + + auto extents = ptr->extents(); + + // Destroy the fiemap, closing file handles. This should not delete them. + ptr = nullptr; + + std::vector files; + ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files)); + for (const auto& path : files) { + EXPECT_EQ(access(path.c_str(), F_OK), 0); + } + + ASSERT_GE(extents.size(), files.size()); +} + +TEST_F(SplitFiemapTest, Open) { + { + auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32); + ASSERT_NE(ptr, nullptr); + } + + auto ptr = SplitFiemap::Open(testfile); + ASSERT_NE(ptr, nullptr); + + auto extents = ptr->extents(); + ASSERT_GE(extents.size(), 24); +} + +TEST_F(SplitFiemapTest, DeleteOnFail) { + auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 10, 1); + ASSERT_EQ(ptr, nullptr); + + std::string first_file = testfile + ".0001"; + ASSERT_NE(access(first_file.c_str(), F_OK), 0); + ASSERT_EQ(errno, ENOENT); + ASSERT_NE(access(testfile.c_str(), F_OK), 0); + ASSERT_EQ(errno, ENOENT); +} + class VerifyBlockWritesExt4 : public ::testing::Test { // 2GB Filesystem and 4k block size by default static constexpr uint64_t block_size = 4096; diff --git a/fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h b/fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h new file mode 100644 index 000000000..765cc8401 --- /dev/null +++ b/fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include +#include +#include + +#include "fiemap_writer.h" + +namespace android { +namespace fiemap_writer { + +// Wrapper around FiemapWriter that is able to split images across files if +// necessary. +class SplitFiemap final { + public: + using ProgressCallback = std::function; + + // Create a new split fiemap file. If |max_piece_size| is 0, the number of + // pieces will be determined automatically by detecting the filesystem. + // Otherwise, the file will be split evenly (with the remainder in the + // final file). + static std::unique_ptr Create(const std::string& file_path, uint64_t file_size, + uint64_t max_piece_size, + ProgressCallback progress = {}); + + // Open an existing split fiemap file. + static std::unique_ptr Open(const std::string& file_path); + + ~SplitFiemap(); + + // Return a list of all files created for a split file. + static bool GetSplitFileList(const std::string& file_path, std::vector* list); + + // Destroy all components of a split file. If the root file does not exist, + // this returns true and does not report an error. + static bool RemoveSplitFiles(const std::string& file_path, std::string* message = nullptr); + + const std::vector& extents(); + uint32_t block_size() const; + uint64_t size() const { return total_size_; } + + // Non-copyable & Non-movable + SplitFiemap(const SplitFiemap&) = delete; + SplitFiemap& operator=(const SplitFiemap&) = delete; + SplitFiemap& operator=(SplitFiemap&&) = delete; + SplitFiemap(SplitFiemap&&) = delete; + + private: + SplitFiemap() = default; + void AddFile(FiemapUniquePtr&& file); + + bool creating_ = false; + std::string list_file_; + std::vector files_; + std::vector extents_; + uint64_t total_size_ = 0; +}; + +} // namespace fiemap_writer +} // namespace android diff --git a/fs_mgr/libfiemap_writer/split_fiemap_writer.cpp b/fs_mgr/libfiemap_writer/split_fiemap_writer.cpp new file mode 100644 index 000000000..1f8037074 --- /dev/null +++ b/fs_mgr/libfiemap_writer/split_fiemap_writer.cpp @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "utility.h" + +namespace android { +namespace fiemap_writer { + +using android::base::unique_fd; + +// We use a four-digit suffix at the end of filenames. +static const size_t kMaxFilePieces = 500; + +std::unique_ptr SplitFiemap::Create(const std::string& file_path, uint64_t file_size, + uint64_t max_piece_size, + ProgressCallback progress) { + if (!file_size) { + LOG(ERROR) << "Cannot create a fiemap for a 0-length file: " << file_path; + return nullptr; + } + + if (!max_piece_size) { + max_piece_size = DetermineMaximumFileSize(file_path); + if (!max_piece_size) { + LOG(ERROR) << "Could not determine maximum file size for " << file_path; + return nullptr; + } + } + + // Call |progress| only when the total percentage would significantly change. + int permille = -1; + uint64_t total_bytes_written = 0; + auto on_progress = [&](uint64_t written, uint64_t) -> bool { + uint64_t actual_written = total_bytes_written + written; + int new_permille = (actual_written * 1000) / file_size; + if (new_permille != permille && actual_written < file_size) { + if (progress && !progress(actual_written, file_size)) { + return false; + } + permille = new_permille; + } + return true; + }; + + std::unique_ptr out(new SplitFiemap()); + out->creating_ = true; + out->list_file_ = file_path; + + // Create the split files. + uint64_t remaining_bytes = file_size; + while (remaining_bytes) { + if (out->files_.size() >= kMaxFilePieces) { + LOG(ERROR) << "Requested size " << file_size << " created too many split files"; + return nullptr; + } + std::string chunk_path = + android::base::StringPrintf("%s.%04d", file_path.c_str(), (int)out->files_.size()); + uint64_t chunk_size = std::min(max_piece_size, remaining_bytes); + auto writer = FiemapWriter::Open(chunk_path, chunk_size, true, on_progress); + if (!writer) { + return nullptr; + } + + // To make sure the alignment doesn't create too much inconsistency, we + // account the *actual* size, not the requested size. + total_bytes_written += writer->size(); + remaining_bytes -= writer->size(); + + out->AddFile(std::move(writer)); + } + + // Create the split file list. + unique_fd fd(open(out->list_file_.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0660)); + if (fd < 0) { + PLOG(ERROR) << "Failed to open " << file_path; + return nullptr; + } + + for (const auto& writer : out->files_) { + std::string line = android::base::Basename(writer->file_path()) + "\n"; + if (!android::base::WriteFully(fd, line.data(), line.size())) { + PLOG(ERROR) << "Write failed " << file_path; + return nullptr; + } + } + + // Unset this bit, so we don't unlink on destruction. + out->creating_ = false; + return out; +} + +std::unique_ptr SplitFiemap::Open(const std::string& file_path) { + std::vector files; + if (!GetSplitFileList(file_path, &files)) { + return nullptr; + } + + std::unique_ptr out(new SplitFiemap()); + out->list_file_ = file_path; + + for (const auto& file : files) { + auto writer = FiemapWriter::Open(file, 0, false); + if (!writer) { + // Error was logged in Open(). + return nullptr; + } + out->AddFile(std::move(writer)); + } + return out; +} + +bool SplitFiemap::GetSplitFileList(const std::string& file_path, std::vector* list) { + // This is not the most efficient thing, but it is simple and recovering + // the fiemap/fibmap is much more expensive. + std::string contents; + if (!android::base::ReadFileToString(file_path, &contents, true)) { + PLOG(ERROR) << "Error reading file: " << file_path; + return false; + } + + std::vector names = android::base::Split(contents, "\n"); + std::string dir = android::base::Dirname(file_path); + for (const auto& name : names) { + if (!name.empty()) { + list->emplace_back(dir + "/" + name); + } + } + return true; +} + +bool SplitFiemap::RemoveSplitFiles(const std::string& file_path, std::string* message) { + // Early exit if this does not exist, and do not report an error. + if (access(file_path.c_str(), F_OK) && errno == ENOENT) { + return true; + } + + bool ok = true; + std::vector files; + if (GetSplitFileList(file_path, &files)) { + for (const auto& file : files) { + ok &= android::base::RemoveFileIfExists(file, message); + } + } + ok &= android::base::RemoveFileIfExists(file_path, message); + return ok; +} + +const std::vector& SplitFiemap::extents() { + if (extents_.empty()) { + for (const auto& file : files_) { + const auto& extents = file->extents(); + extents_.insert(extents_.end(), extents.begin(), extents.end()); + } + } + return extents_; +} + +SplitFiemap::~SplitFiemap() { + if (!creating_) { + return; + } + + // We failed to finish creating, so unlink everything. + unlink(list_file_.c_str()); + for (auto&& file : files_) { + std::string path = file->file_path(); + file = nullptr; + + unlink(path.c_str()); + } +} + +void SplitFiemap::AddFile(FiemapUniquePtr&& file) { + total_size_ += file->size(); + files_.emplace_back(std::move(file)); +} + +uint32_t SplitFiemap::block_size() const { + return files_[0]->block_size(); +} + +} // namespace fiemap_writer +} // namespace android diff --git a/fs_mgr/libfiemap_writer/utility.cpp b/fs_mgr/libfiemap_writer/utility.cpp new file mode 100644 index 000000000..192ec1602 --- /dev/null +++ b/fs_mgr/libfiemap_writer/utility.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utility.h" + +#include +#include +#include + +#include +#include + +namespace android { +namespace fiemap_writer { + +uint64_t DetermineMaximumFileSize(const std::string& file_path) { + // Create the smallest file possible (one block). + auto writer = FiemapWriter::Open(file_path, 1); + if (!writer) { + return 0; + } + + uint64_t result = 0; + switch (writer->fs_type()) { + case EXT4_SUPER_MAGIC: + // The minimum is 16GiB, so just report that. If we wanted we could parse the + // superblock and figure out if 64-bit support is enabled. + result = 17179869184ULL; + break; + case F2FS_SUPER_MAGIC: + // Formula is from https://www.kernel.org/doc/Documentation/filesystems/f2fs.txt + // 4KB * (923 + 2 * 1018 + 2 * 1018 * 1018 + 1018 * 1018 * 1018) := 3.94TB. + result = 4329690886144ULL; + break; + case MSDOS_SUPER_MAGIC: + // 4GB-1, which we want aligned to the block size. + result = 4294967295; + result -= (result % writer->block_size()); + break; + default: + LOG(ERROR) << "Unknown file system type: " << writer->fs_type(); + break; + } + + // Close and delete the temporary file. + writer = nullptr; + unlink(file_path.c_str()); + + return result; +} + +} // namespace fiemap_writer +} // namespace android diff --git a/fs_mgr/libfiemap_writer/utility.h b/fs_mgr/libfiemap_writer/utility.h new file mode 100644 index 000000000..2d418dae4 --- /dev/null +++ b/fs_mgr/libfiemap_writer/utility.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace android { +namespace fiemap_writer { + +// Given a file that will be created, determine the maximum size its containing +// filesystem allows. Note this is a theoretical maximum size; free space is +// ignored entirely. +uint64_t DetermineMaximumFileSize(const std::string& file_path); + +} // namespace fiemap_writer +} // namespace android