From 4d4db8c09e6dba4368112f6647b9b4b986fb5507 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Mon, 25 Feb 2019 13:17:36 -0800 Subject: [PATCH] Add more filesystem support to libfiemap_writer. This adds support for vfat filesystems. Bug: 126230649 Test: fiemap_writer_test gtest Change-Id: I028cc7d95c313dd3ed826bc44cc3d0ffdcb58597 --- fs_mgr/libfiemap_writer/Android.bp | 4 +- fs_mgr/libfiemap_writer/fiemap_writer.cpp | 130 ++++++++++++++---- .../libfiemap_writer/fiemap_writer_test.cpp | 28 ++-- .../include/libfiemap_writer/fiemap_writer.h | 6 +- 4 files changed, 129 insertions(+), 39 deletions(-) diff --git a/fs_mgr/libfiemap_writer/Android.bp b/fs_mgr/libfiemap_writer/Android.bp index 33c3cadcf..f4d03eccb 100644 --- a/fs_mgr/libfiemap_writer/Android.bp +++ b/fs_mgr/libfiemap_writer/Android.bp @@ -20,9 +20,7 @@ cc_library_static { recovery_available: true, export_include_dirs: ["include"], cflags: [ - // TODO(b/121211685): Allows us to create a skeleton of required classes - "-Wno-unused-private-field", - "-Wno-unused-parameter", + "-D_FILE_OFFSET_BITS=64", ], srcs: [ diff --git a/fs_mgr/libfiemap_writer/fiemap_writer.cpp b/fs_mgr/libfiemap_writer/fiemap_writer.cpp index 8b880e6ee..9aa56e170 100644 --- a/fs_mgr/libfiemap_writer/fiemap_writer.cpp +++ b/fs_mgr/libfiemap_writer/fiemap_writer.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -206,10 +207,15 @@ static bool PerformFileChecks(const std::string& file_path, uint64_t file_size, } // Check if the filesystem is of supported types. - // Only ext4 and f2fs are tested and supported. - if ((sfs.f_type != EXT4_SUPER_MAGIC) && (sfs.f_type != F2FS_SUPER_MAGIC)) { - LOG(ERROR) << "Unsupported file system type: 0x" << std::hex << sfs.f_type; - return false; + // Only ext4, f2fs, and vfat are tested and supported. + switch (sfs.f_type) { + case EXT4_SUPER_MAGIC: + case F2FS_SUPER_MAGIC: + case MSDOS_SUPER_MAGIC: + break; + default: + LOG(ERROR) << "Unsupported file system type: 0x" << std::hex << sfs.f_type; + return false; } uint64_t available_bytes = sfs.f_bsize * sfs.f_bavail; @@ -224,14 +230,46 @@ static bool PerformFileChecks(const std::string& file_path, uint64_t file_size, } static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blocksz, - uint64_t file_size, std::function on_progress) { + uint64_t file_size, unsigned int fs_type, + std::function on_progress) { // Reserve space for the file on the file system and write it out to make sure the extents // don't come back unwritten. Return from this function with the kernel file offset set to 0. // If the filesystem is f2fs, then we also PIN the file on disk to make sure the blocks // aren't moved around. - if (fallocate64(file_fd, FALLOC_FL_ZERO_RANGE, 0, file_size)) { - PLOG(ERROR) << "Failed to allocate space for file: " << file_path << " size: " << file_size; - return false; + switch (fs_type) { + case EXT4_SUPER_MAGIC: + case F2FS_SUPER_MAGIC: + if (fallocate(file_fd, FALLOC_FL_ZERO_RANGE, 0, file_size)) { + PLOG(ERROR) << "Failed to allocate space for file: " << file_path + << " size: " << file_size; + return false; + } + break; + case MSDOS_SUPER_MAGIC: { + // fallocate() is not supported, and not needed, since VFAT does not support holes. + // Instead we can perform a much faster allocation. + auto offset = TEMP_FAILURE_RETRY(lseek(file_fd, file_size - 1, SEEK_SET)); + if (offset < 0) { + PLOG(ERROR) << "Failed to lseek " << file_path; + return false; + } + if (offset != file_size - 1) { + LOG(ERROR) << "Seek returned wrong offset " << offset << " for file " << file_path; + return false; + } + char buffer[] = {0}; + if (!android::base::WriteFully(file_fd, buffer, 1)) { + PLOG(ERROR) << "Write failed: " << file_path; + return false; + } + if (on_progress && !on_progress(file_size, file_size)) { + return false; + } + return true; + } + default: + LOG(ERROR) << "Missing fallocate() support for file system " << fs_type; + return false; } // write zeroes in 'blocksz' byte increments until we reach file_size to make sure the data @@ -286,9 +324,9 @@ static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blo } static bool PinFile(int file_fd, const std::string& file_path, uint32_t fs_type) { - if (fs_type == EXT4_SUPER_MAGIC) { - // No pinning necessary for ext4. The blocks, once allocated, are expected - // to be fixed. + if (fs_type != F2FS_SUPER_MAGIC) { + // No pinning necessary for ext4/msdos. The blocks, once allocated, are + // expected to be fixed. return true; } @@ -323,9 +361,9 @@ static bool PinFile(int file_fd, const std::string& file_path, uint32_t fs_type) } static bool IsFilePinned(int file_fd, const std::string& file_path, uint32_t fs_type) { - if (fs_type == EXT4_SUPER_MAGIC) { - // No pinning necessary for ext4. The blocks, once allocated, are expected - // to be fixed. + if (fs_type != F2FS_SUPER_MAGIC) { + // No pinning necessary for ext4 or vfat. The blocks, once allocated, + // are expected to be fixed. return true; } @@ -437,6 +475,44 @@ static bool ReadFiemap(int file_fd, const std::string& file_path, return last_extent_seen; } +static bool ReadFibmap(int file_fd, const std::string& file_path, + std::vector* extents) { + struct stat s; + if (fstat(file_fd, &s)) { + PLOG(ERROR) << "Failed to stat " << file_path; + return false; + } + + uint64_t num_blocks = (s.st_size + s.st_blksize - 1) / s.st_blksize; + if (num_blocks > std::numeric_limits::max()) { + LOG(ERROR) << "Too many blocks for FIBMAP (" << num_blocks << ")"; + return false; + } + + for (uint32_t last_block, block_number = 0; block_number < num_blocks; block_number++) { + uint32_t block = block_number; + if (ioctl(file_fd, FIBMAP, &block)) { + PLOG(ERROR) << "Failed to get FIBMAP for file " << file_path; + return false; + } + if (!block) { + LOG(ERROR) << "Logical block " << block_number << " is a hole, which is not supported"; + return false; + } + + if (!extents->empty() && block == last_block + 1) { + extents->back().fe_length++; + } else { + extents->push_back(fiemap_extent{.fe_logical = block_number, + .fe_physical = block, + .fe_length = 1, + .fe_flags = 0}); + } + last_block = block; + } + return true; +} + FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_size, bool create, std::function progress) { // if 'create' is false, open an existing file and do not truncate. @@ -505,7 +581,7 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s } if (create) { - if (!AllocateFile(file_fd, abs_path, blocksz, file_size, std::move(progress))) { + if (!AllocateFile(file_fd, abs_path, blocksz, file_size, fs_type, std::move(progress))) { LOG(ERROR) << "Failed to allocate file: " << abs_path << " of size: " << file_size << " bytes"; cleanup(abs_path, create); @@ -522,10 +598,22 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s // now allocate the FiemapWriter and start setting it up FiemapUniquePtr fmap(new FiemapWriter()); - if (!ReadFiemap(file_fd, abs_path, &fmap->extents_)) { - LOG(ERROR) << "Failed to read fiemap of file: " << abs_path; - cleanup(abs_path, create); - return nullptr; + switch (fs_type) { + case EXT4_SUPER_MAGIC: + case F2FS_SUPER_MAGIC: + if (!ReadFiemap(file_fd, abs_path, &fmap->extents_)) { + LOG(ERROR) << "Failed to read fiemap of file: " << abs_path; + cleanup(abs_path, create); + return nullptr; + } + break; + case MSDOS_SUPER_MAGIC: + if (!ReadFibmap(file_fd, abs_path, &fmap->extents_)) { + LOG(ERROR) << "Failed to read fibmap of file: " << abs_path; + cleanup(abs_path, create); + return nullptr; + } + break; } fmap->file_path_ = abs_path; @@ -541,9 +629,5 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s return fmap; } -bool FiemapWriter::Read(off64_t off, uint8_t* buffer, uint64_t size) { - return false; -} - } // namespace fiemap_writer } // namespace android diff --git a/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp b/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp index d1c0aadd7..e01a0fb61 100644 --- a/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp +++ b/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp @@ -71,8 +71,7 @@ TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) { TEST_F(FiemapWriterTest, CreateUnalignedFile) { // Try creating a file of size 4097 bytes which is guaranteed - // to be unaligned to all known block sizes. The creation must - // fail. + // to be unaligned to all known block sizes. FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize + 1); ASSERT_NE(fptr, nullptr); ASSERT_EQ(fptr->size(), gBlockSize * 2); @@ -87,10 +86,7 @@ TEST_F(FiemapWriterTest, CheckFilePath) { } TEST_F(FiemapWriterTest, CheckProgress) { - std::vector expected{ - 0, - gBlockSize, - }; + std::vector expected; size_t invocations = 0; auto callback = [&](uint64_t done, uint64_t total) -> bool { EXPECT_LT(invocations, expected.size()); @@ -100,9 +96,22 @@ TEST_F(FiemapWriterTest, CheckProgress) { return true; }; + uint32_t fs_type; + { + auto ptr = FiemapWriter::Open(testfile, gBlockSize, true); + ASSERT_NE(ptr, nullptr); + fs_type = ptr->fs_type(); + } + ASSERT_EQ(unlink(testfile.c_str()), 0); + + if (fs_type != MSDOS_SUPER_MAGIC) { + expected.push_back(0); + } + expected.push_back(gBlockSize); + auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback)); EXPECT_NE(ptr, nullptr); - EXPECT_EQ(invocations, 2); + EXPECT_EQ(invocations, expected.size()); } TEST_F(FiemapWriterTest, CheckPinning) { @@ -273,7 +282,10 @@ int main(int argc, char** argv) { cerr << "unable to create tempdir on " << argv[1] << "\n"; exit(EXIT_FAILURE); } - gTestDir = tempdir; + if (!android::base::Realpath(tempdir, &gTestDir)) { + cerr << "unable to find realpath for " << tempdir; + exit(EXIT_FAILURE); + } if (argc > 2) { testfile_size = strtoull(argv[2], NULL, 0); diff --git a/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h b/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h index 3ba68e9f9..831bc75ae 100644 --- a/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h +++ b/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h @@ -67,11 +67,6 @@ class FiemapWriter final { static bool GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path, bool* uses_dm = nullptr); - // The counter part of Write(). It is an error for the offset to be unaligned with - // the block device's block size. - // In case of error, the contents of buffer MUST be discarded. - bool Read(off64_t off, uint8_t* buffer, uint64_t size); - ~FiemapWriter() = default; const std::string& file_path() const { return file_path_; }; @@ -79,6 +74,7 @@ class FiemapWriter final { const std::string& bdev_path() const { return bdev_path_; }; uint64_t block_size() const { return block_size_; }; const std::vector& extents() { return extents_; }; + uint32_t fs_type() const { return fs_type_; } // Non-copyable & Non-movable FiemapWriter(const FiemapWriter&) = delete;