Support FiemapWriters that extend across multiple files.

This introduces a new SplitFiemap class that will divide an allocation
request across multiple FiemapWriters. This is primarily useful on
filesystems that have onerous restrictions on maximum file sizes. Vfat,
for example, supports a maximum of 4GiB, which is too small to satisfy
larger userdata size requests.

Bug: 126230649
Test: fiemap_writer_test gtest
Change-Id: I3c95d341e4e94e0c44bbf0e8553c34ccfdcd155b

fs_mgr/libfiemap_writer/Android.bp

@@ -25,6 +25,12 @@ cc_library_static {
 
     srcs: [
         "fiemap_writer.cpp",
+        "split_fiemap_writer.cpp",
+        "utility.cpp",
+    ],
+
+    static_libs: [
+        "libext4_utils",
     ],
 
     header_libs: [

fs_mgr/libfiemap_writer/fiemap_writer.cpp

@@ -624,8 +624,8 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s
     fmap->fs_type_ = fs_type;
     fmap->block_size_ = blocksz;
 
-    LOG(INFO) << "Successfully created FiemapWriter for file " << abs_path << " on block device "
-              << bdev_path;
+    LOG(VERBOSE) << "Successfully created FiemapWriter for file " << abs_path << " on block device "
+                 << bdev_path;
     return fmap;
 }
 

fs_mgr/libfiemap_writer/fiemap_writer_test.cpp

@@ -33,8 +33,10 @@
 #include <android-base/unique_fd.h>
 #include <gtest/gtest.h>
 #include <libdm/loop_control.h>
-
 #include <libfiemap_writer/fiemap_writer.h>
+#include <libfiemap_writer/split_fiemap_writer.h>
+
+#include "utility.h"
 
 using namespace std;
 using namespace std::string_literals;
@@ -59,6 +61,24 @@ class FiemapWriterTest : public ::testing::Test {
     std::string testfile;
 };
 
+class SplitFiemapTest : public ::testing::Test {
+  protected:
+    void SetUp() override {
+        const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
+        testfile = gTestDir + "/"s + tinfo->name();
+    }
+
+    void TearDown() override {
+        std::string message;
+        if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) {
+            cerr << "Could not remove all split files: " << message;
+        }
+    }
+
+    // name of the file we use for testing
+    std::string testfile;
+};
+
 TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) {
     // Try creating a file of size ~100TB but aligned to
     // 512 byte to make sure block alignment tests don't
@@ -168,6 +188,52 @@ TEST_F(FiemapWriterTest, FileDeletedOnError) {
     EXPECT_EQ(errno, ENOENT);
 }
 
+TEST_F(FiemapWriterTest, MaxBlockSize) {
+    ASSERT_GT(DetermineMaximumFileSize(testfile), 0);
+}
+
+TEST_F(SplitFiemapTest, Create) {
+    auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
+    ASSERT_NE(ptr, nullptr);
+
+    auto extents = ptr->extents();
+
+    // Destroy the fiemap, closing file handles. This should not delete them.
+    ptr = nullptr;
+
+    std::vector<std::string> files;
+    ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files));
+    for (const auto& path : files) {
+        EXPECT_EQ(access(path.c_str(), F_OK), 0);
+    }
+
+    ASSERT_GE(extents.size(), files.size());
+}
+
+TEST_F(SplitFiemapTest, Open) {
+    {
+        auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
+        ASSERT_NE(ptr, nullptr);
+    }
+
+    auto ptr = SplitFiemap::Open(testfile);
+    ASSERT_NE(ptr, nullptr);
+
+    auto extents = ptr->extents();
+    ASSERT_GE(extents.size(), 24);
+}
+
+TEST_F(SplitFiemapTest, DeleteOnFail) {
+    auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 10, 1);
+    ASSERT_EQ(ptr, nullptr);
+
+    std::string first_file = testfile + ".0001";
+    ASSERT_NE(access(first_file.c_str(), F_OK), 0);
+    ASSERT_EQ(errno, ENOENT);
+    ASSERT_NE(access(testfile.c_str(), F_OK), 0);
+    ASSERT_EQ(errno, ENOENT);
+}
+
 class VerifyBlockWritesExt4 : public ::testing::Test {
     // 2GB Filesystem and 4k block size by default
     static constexpr uint64_t block_size = 4096;

fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h

@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "fiemap_writer.h"
+
+namespace android {
+namespace fiemap_writer {
+
+// Wrapper around FiemapWriter that is able to split images across files if
+// necessary.
+class SplitFiemap final {
+  public:
+    using ProgressCallback = std::function<bool(uint64_t, uint64_t)>;
+
+    // Create a new split fiemap file. If |max_piece_size| is 0, the number of
+    // pieces will be determined automatically by detecting the filesystem.
+    // Otherwise, the file will be split evenly (with the remainder in the
+    // final file).
+    static std::unique_ptr<SplitFiemap> Create(const std::string& file_path, uint64_t file_size,
+                                               uint64_t max_piece_size,
+                                               ProgressCallback progress = {});
+
+    // Open an existing split fiemap file.
+    static std::unique_ptr<SplitFiemap> Open(const std::string& file_path);
+
+    ~SplitFiemap();
+
+    // Return a list of all files created for a split file.
+    static bool GetSplitFileList(const std::string& file_path, std::vector<std::string>* list);
+
+    // Destroy all components of a split file. If the root file does not exist,
+    // this returns true and does not report an error.
+    static bool RemoveSplitFiles(const std::string& file_path, std::string* message = nullptr);
+
+    const std::vector<struct fiemap_extent>& extents();
+    uint32_t block_size() const;
+    uint64_t size() const { return total_size_; }
+
+    // Non-copyable & Non-movable
+    SplitFiemap(const SplitFiemap&) = delete;
+    SplitFiemap& operator=(const SplitFiemap&) = delete;
+    SplitFiemap& operator=(SplitFiemap&&) = delete;
+    SplitFiemap(SplitFiemap&&) = delete;
+
+  private:
+    SplitFiemap() = default;
+    void AddFile(FiemapUniquePtr&& file);
+
+    bool creating_ = false;
+    std::string list_file_;
+    std::vector<FiemapUniquePtr> files_;
+    std::vector<struct fiemap_extent> extents_;
+    uint64_t total_size_ = 0;
+};
+
+}  // namespace fiemap_writer
+}  // namespace android

fs_mgr/libfiemap_writer/split_fiemap_writer.cpp

@@ -0,0 +1,214 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <libfiemap_writer/split_fiemap_writer.h>
+
+#include <fcntl.h>
+#include <stdint.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/stringprintf.h>
+#include <android-base/strings.h>
+#include <android-base/unique_fd.h>
+
+#include "utility.h"
+
+namespace android {
+namespace fiemap_writer {
+
+using android::base::unique_fd;
+
+// We use a four-digit suffix at the end of filenames.
+static const size_t kMaxFilePieces = 500;
+
+std::unique_ptr<SplitFiemap> SplitFiemap::Create(const std::string& file_path, uint64_t file_size,
+                                                 uint64_t max_piece_size,
+                                                 ProgressCallback progress) {
+    if (!file_size) {
+        LOG(ERROR) << "Cannot create a fiemap for a 0-length file: " << file_path;
+        return nullptr;
+    }
+
+    if (!max_piece_size) {
+        max_piece_size = DetermineMaximumFileSize(file_path);
+        if (!max_piece_size) {
+            LOG(ERROR) << "Could not determine maximum file size for " << file_path;
+            return nullptr;
+        }
+    }
+
+    // Call |progress| only when the total percentage would significantly change.
+    int permille = -1;
+    uint64_t total_bytes_written = 0;
+    auto on_progress = [&](uint64_t written, uint64_t) -> bool {
+        uint64_t actual_written = total_bytes_written + written;
+        int new_permille = (actual_written * 1000) / file_size;
+        if (new_permille != permille && actual_written < file_size) {
+            if (progress && !progress(actual_written, file_size)) {
+                return false;
+            }
+            permille = new_permille;
+        }
+        return true;
+    };
+
+    std::unique_ptr<SplitFiemap> out(new SplitFiemap());
+    out->creating_ = true;
+    out->list_file_ = file_path;
+
+    // Create the split files.
+    uint64_t remaining_bytes = file_size;
+    while (remaining_bytes) {
+        if (out->files_.size() >= kMaxFilePieces) {
+            LOG(ERROR) << "Requested size " << file_size << " created too many split files";
+            return nullptr;
+        }
+        std::string chunk_path =
+                android::base::StringPrintf("%s.%04d", file_path.c_str(), (int)out->files_.size());
+        uint64_t chunk_size = std::min(max_piece_size, remaining_bytes);
+        auto writer = FiemapWriter::Open(chunk_path, chunk_size, true, on_progress);
+        if (!writer) {
+            return nullptr;
+        }
+
+        // To make sure the alignment doesn't create too much inconsistency, we
+        // account the *actual* size, not the requested size.
+        total_bytes_written += writer->size();
+        remaining_bytes -= writer->size();
+
+        out->AddFile(std::move(writer));
+    }
+
+    // Create the split file list.
+    unique_fd fd(open(out->list_file_.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0660));
+    if (fd < 0) {
+        PLOG(ERROR) << "Failed to open " << file_path;
+        return nullptr;
+    }
+
+    for (const auto& writer : out->files_) {
+        std::string line = android::base::Basename(writer->file_path()) + "\n";
+        if (!android::base::WriteFully(fd, line.data(), line.size())) {
+            PLOG(ERROR) << "Write failed " << file_path;
+            return nullptr;
+        }
+    }
+
+    // Unset this bit, so we don't unlink on destruction.
+    out->creating_ = false;
+    return out;
+}
+
+std::unique_ptr<SplitFiemap> SplitFiemap::Open(const std::string& file_path) {
+    std::vector<std::string> files;
+    if (!GetSplitFileList(file_path, &files)) {
+        return nullptr;
+    }
+
+    std::unique_ptr<SplitFiemap> out(new SplitFiemap());
+    out->list_file_ = file_path;
+
+    for (const auto& file : files) {
+        auto writer = FiemapWriter::Open(file, 0, false);
+        if (!writer) {
+            // Error was logged in Open().
+            return nullptr;
+        }
+        out->AddFile(std::move(writer));
+    }
+    return out;
+}
+
+bool SplitFiemap::GetSplitFileList(const std::string& file_path, std::vector<std::string>* list) {
+    // This is not the most efficient thing, but it is simple and recovering
+    // the fiemap/fibmap is much more expensive.
+    std::string contents;
+    if (!android::base::ReadFileToString(file_path, &contents, true)) {
+        PLOG(ERROR) << "Error reading file: " << file_path;
+        return false;
+    }
+
+    std::vector<std::string> names = android::base::Split(contents, "\n");
+    std::string dir = android::base::Dirname(file_path);
+    for (const auto& name : names) {
+        if (!name.empty()) {
+            list->emplace_back(dir + "/" + name);
+        }
+    }
+    return true;
+}
+
+bool SplitFiemap::RemoveSplitFiles(const std::string& file_path, std::string* message) {
+    // Early exit if this does not exist, and do not report an error.
+    if (access(file_path.c_str(), F_OK) && errno == ENOENT) {
+        return true;
+    }
+
+    bool ok = true;
+    std::vector<std::string> files;
+    if (GetSplitFileList(file_path, &files)) {
+        for (const auto& file : files) {
+            ok &= android::base::RemoveFileIfExists(file, message);
+        }
+    }
+    ok &= android::base::RemoveFileIfExists(file_path, message);
+    return ok;
+}
+
+const std::vector<struct fiemap_extent>& SplitFiemap::extents() {
+    if (extents_.empty()) {
+        for (const auto& file : files_) {
+            const auto& extents = file->extents();
+            extents_.insert(extents_.end(), extents.begin(), extents.end());
+        }
+    }
+    return extents_;
+}
+
+SplitFiemap::~SplitFiemap() {
+    if (!creating_) {
+        return;
+    }
+
+    // We failed to finish creating, so unlink everything.
+    unlink(list_file_.c_str());
+    for (auto&& file : files_) {
+        std::string path = file->file_path();
+        file = nullptr;
+
+        unlink(path.c_str());
+    }
+}
+
+void SplitFiemap::AddFile(FiemapUniquePtr&& file) {
+    total_size_ += file->size();
+    files_.emplace_back(std::move(file));
+}
+
+uint32_t SplitFiemap::block_size() const {
+    return files_[0]->block_size();
+}
+
+}  // namespace fiemap_writer
+}  // namespace android

fs_mgr/libfiemap_writer/utility.cpp

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "utility.h"
+
+#include <stdint.h>
+#include <sys/vfs.h>
+#include <unistd.h>
+
+#include <android-base/logging.h>
+#include <libfiemap_writer/fiemap_writer.h>
+
+namespace android {
+namespace fiemap_writer {
+
+uint64_t DetermineMaximumFileSize(const std::string& file_path) {
+    // Create the smallest file possible (one block).
+    auto writer = FiemapWriter::Open(file_path, 1);
+    if (!writer) {
+        return 0;
+    }
+
+    uint64_t result = 0;
+    switch (writer->fs_type()) {
+        case EXT4_SUPER_MAGIC:
+            // The minimum is 16GiB, so just report that. If we wanted we could parse the
+            // superblock and figure out if 64-bit support is enabled.
+            result = 17179869184ULL;
+            break;
+        case F2FS_SUPER_MAGIC:
+            // Formula is from https://www.kernel.org/doc/Documentation/filesystems/f2fs.txt
+            // 4KB * (923 + 2 * 1018 + 2 * 1018 * 1018 + 1018 * 1018 * 1018) := 3.94TB.
+            result = 4329690886144ULL;
+            break;
+        case MSDOS_SUPER_MAGIC:
+            // 4GB-1, which we want aligned to the block size.
+            result = 4294967295;
+            result -= (result % writer->block_size());
+            break;
+        default:
+            LOG(ERROR) << "Unknown file system type: " << writer->fs_type();
+            break;
+    }
+
+    // Close and delete the temporary file.
+    writer = nullptr;
+    unlink(file_path.c_str());
+
+    return result;
+}
+
+}  // namespace fiemap_writer
+}  // namespace android

fs_mgr/libfiemap_writer/utility.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+namespace android {
+namespace fiemap_writer {
+
+// Given a file that will be created, determine the maximum size its containing
+// filesystem allows. Note this is a theoretical maximum size; free space is
+// ignored entirely.
+uint64_t DetermineMaximumFileSize(const std::string& file_path);
+
+}  // namespace fiemap_writer
+}  // namespace android