diff --git a/fs_mgr/libfiemap_writer/.clang-format b/fs_mgr/libfiemap_writer/.clang-format deleted file mode 120000 index 8b770a15c..000000000 --- a/fs_mgr/libfiemap_writer/.clang-format +++ /dev/null @@ -1 +0,0 @@ -../../.clang-format-4 \ No newline at end of file diff --git a/fs_mgr/libfiemap_writer/Android.bp b/fs_mgr/libfiemap_writer/Android.bp deleted file mode 100644 index ed209aa9c..000000000 --- a/fs_mgr/libfiemap_writer/Android.bp +++ /dev/null @@ -1,64 +0,0 @@ -// -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -cc_library_static { - name: "libfiemap_writer", - defaults: ["fs_mgr_defaults"], - recovery_available: true, - export_include_dirs: ["include"], - cflags: [ - "-D_FILE_OFFSET_BITS=64", - ], - - srcs: [ - "fiemap_writer.cpp", - "split_fiemap_writer.cpp", - "utility.cpp", - ], - - static_libs: [ - "libdm", - "libext4_utils", - ], - - header_libs: [ - "libbase_headers", - "liblog_headers", - ], -} - -cc_test { - name: "fiemap_writer_test", - cflags: [ - "-D_FILE_OFFSET_BITS=64", - ], - static_libs: [ - "libbase", - "libdm", - "libfiemap_writer", - "liblog", - ], - - data: [ - "testdata/unaligned_file", - "testdata/file_4k", - "testdata/file_32k", - ], - - srcs: [ - "fiemap_writer_test.cpp", - ], -} diff --git a/fs_mgr/libfiemap_writer/Android.mk b/fs_mgr/libfiemap_writer/Android.mk deleted file mode 100644 index 3c07b8ea6..000000000 --- a/fs_mgr/libfiemap_writer/Android.mk +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (C) 2019 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -LOCAL_PATH := $(call my-dir) - -include $(CLEAR_VARS) - -LOCAL_MODULE := VtsFiemapWriterTest --include test/vts/tools/build/Android.host_config.mk diff --git a/fs_mgr/libfiemap_writer/AndroidTest.xml b/fs_mgr/libfiemap_writer/AndroidTest.xml deleted file mode 100644 index 08cff0e24..000000000 --- a/fs_mgr/libfiemap_writer/AndroidTest.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - diff --git a/fs_mgr/libfiemap_writer/fiemap_writer.cpp b/fs_mgr/libfiemap_writer/fiemap_writer.cpp deleted file mode 100644 index 0a3ba6c24..000000000 --- a/fs_mgr/libfiemap_writer/fiemap_writer.cpp +++ /dev/null @@ -1,745 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace android { -namespace fiemap_writer { - -using namespace android::dm; - -// We are expecting no more than 512 extents in a fiemap of the file we create. -// If we find more, then it is treated as error for now. -static constexpr const uint32_t kMaxExtents = 512; - -// TODO: Fallback to using fibmap if FIEMAP_EXTENT_MERGED is set. -static constexpr const uint32_t kUnsupportedExtentFlags = - FIEMAP_EXTENT_UNKNOWN | FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_DELALLOC | - FIEMAP_EXTENT_NOT_ALIGNED | FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_DATA_TAIL | - FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_SHARED | FIEMAP_EXTENT_MERGED; - -// Large file support must be enabled. -static_assert(sizeof(off_t) == sizeof(uint64_t)); - -static inline void cleanup(const std::string& file_path, bool created) { - if (created) { - unlink(file_path.c_str()); - } -} - -static bool BlockDeviceToName(uint32_t major, uint32_t minor, std::string* bdev_name) { - // The symlinks in /sys/dev/block point to the block device node under /sys/device/.. - // The directory name in the target corresponds to the name of the block device. We use - // that to extract the block device name. - // e.g for block device name 'ram0', there exists a symlink named '1:0' in /sys/dev/block as - // follows. - // 1:0 -> ../../devices/virtual/block/ram0 - std::string sysfs_path = ::android::base::StringPrintf("/sys/dev/block/%u:%u", major, minor); - std::string sysfs_bdev; - - if (!::android::base::Readlink(sysfs_path, &sysfs_bdev)) { - PLOG(ERROR) << "Failed to read link at: " << sysfs_path; - return false; - } - - *bdev_name = ::android::base::Basename(sysfs_bdev); - // Paranoid sanity check to make sure we just didn't get the - // input in return as-is. - if (sysfs_bdev == *bdev_name) { - LOG(ERROR) << "Malformed symlink for block device: " << sysfs_bdev; - return false; - } - - return true; -} - -static bool ValidateDmTarget(const DeviceMapper::TargetInfo& target) { - const auto& entry = target.spec; - if (entry.sector_start != 0) { - LOG(INFO) << "Stopping at target with non-zero starting sector"; - return false; - } - - auto target_type = DeviceMapper::GetTargetType(entry); - if (target_type == "bow" || target_type == "default-key" || target_type == "crypt") { - return true; - } - if (target_type == "linear") { - auto pieces = android::base::Split(target.data, " "); - if (pieces[1] != "0") { - LOG(INFO) << "Stopping at complex linear target with non-zero starting sector: " - << pieces[1]; - return false; - } - return true; - } - - LOG(INFO) << "Stopping at complex target type " << target_type; - return false; -} - -static bool DeviceMapperStackPop(const std::string& bdev, std::string* bdev_raw) { - *bdev_raw = bdev; - - if (!::android::base::StartsWith(bdev, "dm-")) { - // We are at the bottom of the device mapper stack. - return true; - } - - // Get the device name. - auto dm_name_file = "/sys/block/" + bdev + "/dm/name"; - std::string dm_name; - if (!android::base::ReadFileToString(dm_name_file, &dm_name)) { - PLOG(ERROR) << "Could not read file: " << dm_name_file; - return false; - } - dm_name = android::base::Trim(dm_name); - - auto& dm = DeviceMapper::Instance(); - std::vector table; - if (!dm.GetTableInfo(dm_name, &table)) { - LOG(ERROR) << "Could not read device-mapper table for " << dm_name << " at " << bdev; - return false; - } - - // The purpose of libfiemap_writer is to provide an extent-based view into - // a file. This is difficult if devices are not layered in a 1:1 manner; - // we would have to translate and break up extents based on the actual - // block mapping. Since this is too complex, we simply stop processing - // the device-mapper stack if we encounter a complex case. - // - // It is up to the caller to decide whether stopping at a virtual block - // device is allowable. In most cases it is not, because we want either - // "userdata" or an external volume. It is useful for tests however. - // Callers can check by comparing the device number to that of userdata, - // or by checking whether is a device-mapper node. - if (table.size() > 1) { - LOG(INFO) << "Stopping at complex table for " << dm_name << " at " << bdev; - return true; - } - if (!ValidateDmTarget(table[0])) { - return true; - } - - auto dm_leaf_dir = "/sys/block/" + bdev + "/slaves"; - auto d = std::unique_ptr(opendir(dm_leaf_dir.c_str()), closedir); - if (d == nullptr) { - PLOG(ERROR) << "Failed to open: " << dm_leaf_dir; - return false; - } - - struct dirent* de; - uint32_t num_leaves = 0; - std::string bdev_next = ""; - while ((de = readdir(d.get())) != nullptr) { - if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) { - continue; - } - - // We set the first name we find here - if (bdev_next.empty()) { - bdev_next = de->d_name; - } - num_leaves++; - } - - // if we have more than one leaves, we return immediately. We can't continue to create the - // file since we don't know how to write it out using fiemap, so it will be readable via the - // underlying block devices later. The reader will also have to construct the same device mapper - // target in order read the file out. - if (num_leaves > 1) { - LOG(ERROR) << "Found " << num_leaves << " leaf block devices under device mapper device " - << bdev; - return false; - } - - // recursively call with the block device we found in order to pop the device mapper stack. - return DeviceMapperStackPop(bdev_next, bdev_raw); -} - -bool FiemapWriter::GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path, - bool* uses_dm) { - struct stat sb; - if (stat(file_path.c_str(), &sb)) { - PLOG(ERROR) << "Failed to get stat for: " << file_path; - return false; - } - - std::string bdev; - if (!BlockDeviceToName(major(sb.st_dev), minor(sb.st_dev), &bdev)) { - LOG(ERROR) << "Failed to get block device name for " << major(sb.st_dev) << ":" - << minor(sb.st_dev); - return false; - } - - std::string bdev_raw; - if (!DeviceMapperStackPop(bdev, &bdev_raw)) { - LOG(ERROR) << "Failed to get the bottom of the device mapper stack for device: " << bdev; - return false; - } - - if (uses_dm) { - *uses_dm = (bdev_raw != bdev); - } - - LOG(DEBUG) << "Popped device (" << bdev_raw << ") from device mapper stack starting with (" - << bdev << ")"; - - *bdev_path = ::android::base::StringPrintf("/dev/block/%s", bdev_raw.c_str()); - - // Make sure we are talking to a block device before calling it a success. - if (stat(bdev_path->c_str(), &sb)) { - PLOG(ERROR) << "Failed to get stat for block device: " << *bdev_path; - return false; - } - - if ((sb.st_mode & S_IFMT) != S_IFBLK) { - PLOG(ERROR) << "File: " << *bdev_path << " is not a block device"; - return false; - } - - return true; -} - -static bool GetBlockDeviceSize(int bdev_fd, const std::string& bdev_path, uint64_t* bdev_size) { - uint64_t size_in_bytes = 0; - if (ioctl(bdev_fd, BLKGETSIZE64, &size_in_bytes)) { - PLOG(ERROR) << "Failed to get total size for: " << bdev_path; - return false; - } - - *bdev_size = size_in_bytes; - - return true; -} - -static uint64_t GetFileSize(const std::string& file_path) { - struct stat sb; - if (stat(file_path.c_str(), &sb)) { - PLOG(ERROR) << "Failed to get size for file: " << file_path; - return 0; - } - - return sb.st_size; -} - -static bool PerformFileChecks(const std::string& file_path, uint64_t file_size, uint64_t* blocksz, - uint32_t* fs_type) { - struct statfs64 sfs; - if (statfs64(file_path.c_str(), &sfs)) { - PLOG(ERROR) << "Failed to read file system status at: " << file_path; - return false; - } - - if (!sfs.f_bsize) { - LOG(ERROR) << "Unsupported block size: " << sfs.f_bsize; - return false; - } - - // Check if the filesystem is of supported types. - // Only ext4, f2fs, and vfat are tested and supported. - switch (sfs.f_type) { - case EXT4_SUPER_MAGIC: - case F2FS_SUPER_MAGIC: - case MSDOS_SUPER_MAGIC: - break; - default: - LOG(ERROR) << "Unsupported file system type: 0x" << std::hex << sfs.f_type; - return false; - } - - uint64_t available_bytes = sfs.f_bsize * sfs.f_bavail; - if (available_bytes <= file_size) { - LOG(ERROR) << "Not enough free space in file system to create file of size : " << file_size; - return false; - } - - *blocksz = sfs.f_bsize; - *fs_type = sfs.f_type; - return true; -} - -static bool FallocateFallback(int file_fd, uint64_t block_size, uint64_t file_size, - const std::string& file_path, - const std::function& on_progress) { - // Even though this is much faster than writing zeroes, it is still slow - // enough that we need to fire the progress callback periodically. To - // easily achieve this, we seek in chunks. We use 1000 chunks since - // normally we only fire the callback on 1/1000th increments. - uint64_t bytes_per_chunk = std::max(file_size / 1000, block_size); - - // Seek just to the end of each chunk and write a single byte, causing - // the filesystem to allocate blocks. - off_t cursor = 0; - off_t end = static_cast(file_size); - while (cursor < end) { - cursor = std::min(static_cast(cursor + bytes_per_chunk), end); - auto rv = TEMP_FAILURE_RETRY(lseek(file_fd, cursor - 1, SEEK_SET)); - if (rv < 0) { - PLOG(ERROR) << "Failed to lseek " << file_path; - return false; - } - if (rv != cursor - 1) { - LOG(ERROR) << "Seek returned wrong offset " << rv << " for file " << file_path; - return false; - } - char buffer[] = {0}; - if (!android::base::WriteFully(file_fd, buffer, 1)) { - PLOG(ERROR) << "Write failed: " << file_path; - return false; - } - if (on_progress && !on_progress(cursor, file_size)) { - return false; - } - } - return true; -} - -static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blocksz, - uint64_t file_size, unsigned int fs_type, - std::function on_progress) { - // Reserve space for the file on the file system and write it out to make sure the extents - // don't come back unwritten. Return from this function with the kernel file offset set to 0. - // If the filesystem is f2fs, then we also PIN the file on disk to make sure the blocks - // aren't moved around. - switch (fs_type) { - case EXT4_SUPER_MAGIC: - case F2FS_SUPER_MAGIC: - if (fallocate(file_fd, FALLOC_FL_ZERO_RANGE, 0, file_size)) { - PLOG(ERROR) << "Failed to allocate space for file: " << file_path - << " size: " << file_size; - return false; - } - break; - case MSDOS_SUPER_MAGIC: - // fallocate() is not supported, and not needed, since VFAT does not support holes. - // Instead we can perform a much faster allocation. - return FallocateFallback(file_fd, blocksz, file_size, file_path, on_progress); - default: - LOG(ERROR) << "Missing fallocate() support for file system " << fs_type; - return false; - } - - // write zeroes in 'blocksz' byte increments until we reach file_size to make sure the data - // blocks are actually written to by the file system and thus getting rid of the holes in the - // file. - auto buffer = std::unique_ptr(calloc(1, blocksz), free); - if (buffer == nullptr) { - LOG(ERROR) << "failed to allocate memory for writing file"; - return false; - } - - off64_t offset = lseek64(file_fd, 0, SEEK_SET); - if (offset < 0) { - PLOG(ERROR) << "Failed to seek at the beginning of : " << file_path; - return false; - } - - int permille = -1; - while (offset < file_size) { - if (!::android::base::WriteFully(file_fd, buffer.get(), blocksz)) { - PLOG(ERROR) << "Failed to write" << blocksz << " bytes at offset" << offset - << " in file " << file_path; - return false; - } - - offset += blocksz; - - // Don't invoke the callback every iteration - wait until a significant - // chunk (here, 1/1000th) of the data has been processed. - int new_permille = (static_cast(offset) * 1000) / file_size; - if (new_permille != permille && static_cast(offset) != file_size) { - if (on_progress && !on_progress(offset, file_size)) { - return false; - } - permille = new_permille; - } - } - - if (lseek64(file_fd, 0, SEEK_SET) < 0) { - PLOG(ERROR) << "Failed to reset offset at the beginning of : " << file_path; - return false; - } - - // flush all writes here .. - if (fsync(file_fd)) { - PLOG(ERROR) << "Failed to synchronize written file:" << file_path; - return false; - } - - // Send one last progress notification. - if (on_progress && !on_progress(file_size, file_size)) { - return false; - } - return true; -} - -static bool PinFile(int file_fd, const std::string& file_path, uint32_t fs_type) { - if (fs_type != F2FS_SUPER_MAGIC) { - // No pinning necessary for ext4/msdos. The blocks, once allocated, are - // expected to be fixed. - return true; - } - -// F2FS-specific ioctl -// It requires the below kernel commit merged in v4.16-rc1. -// 1ad71a27124c ("f2fs: add an ioctl to disable GC for specific file") -// In android-4.4, -// 56ee1e817908 ("f2fs: updates on v4.16-rc1") -// In android-4.9, -// 2f17e34672a8 ("f2fs: updates on v4.16-rc1") -// In android-4.14, -// ce767d9a55bc ("f2fs: updates on v4.16-rc1") -#ifndef F2FS_IOC_SET_PIN_FILE -#ifndef F2FS_IOCTL_MAGIC -#define F2FS_IOCTL_MAGIC 0xf5 -#endif -#define F2FS_IOC_SET_PIN_FILE _IOW(F2FS_IOCTL_MAGIC, 13, __u32) -#endif - - uint32_t pin_status = 1; - int error = ioctl(file_fd, F2FS_IOC_SET_PIN_FILE, &pin_status); - if (error < 0) { - if ((errno == ENOTTY) || (errno == ENOTSUP)) { - PLOG(ERROR) << "Failed to pin file, not supported by kernel: " << file_path; - } else { - PLOG(ERROR) << "Failed to pin file: " << file_path; - } - return false; - } - - return true; -} - -static bool IsFilePinned(int file_fd, const std::string& file_path, uint32_t fs_type) { - if (fs_type != F2FS_SUPER_MAGIC) { - // No pinning necessary for ext4 or vfat. The blocks, once allocated, - // are expected to be fixed. - return true; - } - -// F2FS-specific ioctl -// It requires the below kernel commit merged in v4.16-rc1. -// 1ad71a27124c ("f2fs: add an ioctl to disable GC for specific file") -// In android-4.4, -// 56ee1e817908 ("f2fs: updates on v4.16-rc1") -// In android-4.9, -// 2f17e34672a8 ("f2fs: updates on v4.16-rc1") -// In android-4.14, -// ce767d9a55bc ("f2fs: updates on v4.16-rc1") -#ifndef F2FS_IOC_GET_PIN_FILE -#ifndef F2FS_IOCTL_MAGIC -#define F2FS_IOCTL_MAGIC 0xf5 -#endif -#define F2FS_IOC_GET_PIN_FILE _IOR(F2FS_IOCTL_MAGIC, 14, __u32) -#endif - - // f2fs: export FS_NOCOW_FL flag to user - uint32_t flags; - int error = ioctl(file_fd, FS_IOC_GETFLAGS, &flags); - if (error < 0) { - if ((errno == ENOTTY) || (errno == ENOTSUP)) { - PLOG(ERROR) << "Failed to get flags, not supported by kernel: " << file_path; - } else { - PLOG(ERROR) << "Failed to get flags: " << file_path; - } - return false; - } - if (!(flags & FS_NOCOW_FL)) { - LOG(ERROR) << "It is not pinned: " << file_path; - return false; - } - - // F2FS_IOC_GET_PIN_FILE returns the number of blocks moved. - uint32_t moved_blocks_nr; - error = ioctl(file_fd, F2FS_IOC_GET_PIN_FILE, &moved_blocks_nr); - if (error < 0) { - if ((errno == ENOTTY) || (errno == ENOTSUP)) { - PLOG(ERROR) << "Failed to get file pin status, not supported by kernel: " << file_path; - } else { - PLOG(ERROR) << "Failed to get file pin status: " << file_path; - } - return false; - } - - if (moved_blocks_nr) { - LOG(ERROR) << moved_blocks_nr << " blocks moved in file " << file_path; - } - return moved_blocks_nr == 0; -} - -bool FiemapWriter::HasPinnedExtents(const std::string& file_path) { - android::base::unique_fd fd(open(file_path.c_str(), O_NOFOLLOW | O_CLOEXEC | O_RDONLY)); - if (fd < 0) { - PLOG(ERROR) << "open: " << file_path; - return false; - } - - struct statfs64 sfs; - if (fstatfs64(fd, &sfs)) { - PLOG(ERROR) << "fstatfs64: " << file_path; - return false; - } - return IsFilePinned(fd, file_path, sfs.f_type); -} - -static bool ReadFiemap(int file_fd, const std::string& file_path, - std::vector* extents) { - uint64_t fiemap_size = - sizeof(struct fiemap_extent) + kMaxExtents * sizeof(struct fiemap_extent); - auto buffer = std::unique_ptr(calloc(1, fiemap_size), free); - if (buffer == nullptr) { - LOG(ERROR) << "Failed to allocate memory for fiemap"; - return false; - } - - struct fiemap* fiemap = reinterpret_cast(buffer.get()); - fiemap->fm_start = 0; - fiemap->fm_length = UINT64_MAX; - // make sure file is synced to disk before we read the fiemap - fiemap->fm_flags = FIEMAP_FLAG_SYNC; - fiemap->fm_extent_count = kMaxExtents; - - if (ioctl(file_fd, FS_IOC_FIEMAP, fiemap)) { - PLOG(ERROR) << "Failed to get FIEMAP from the kernel for file: " << file_path; - return false; - } - - if (fiemap->fm_mapped_extents == 0) { - LOG(ERROR) << "File " << file_path << " has zero extents"; - return false; - } - - // Iterate through each extent read and make sure its valid before adding it to the vector - bool last_extent_seen = false; - struct fiemap_extent* extent = &fiemap->fm_extents[0]; - for (uint32_t i = 0; i < fiemap->fm_mapped_extents; i++, extent++) { - // LogExtent(i + 1, *extent); - if (extent->fe_flags & kUnsupportedExtentFlags) { - LOG(ERROR) << "Extent " << i + 1 << " of file " << file_path - << " has unsupported flags"; - extents->clear(); - return false; - } - - if (extent->fe_flags & FIEMAP_EXTENT_LAST) { - last_extent_seen = true; - if (i != (fiemap->fm_mapped_extents - 1)) { - LOG(WARNING) << "Extents are being received out-of-order"; - } - } - extents->emplace_back(std::move(*extent)); - } - - if (!last_extent_seen) { - // The file is possibly too fragmented. - if (fiemap->fm_mapped_extents == kMaxExtents) { - LOG(ERROR) << "File is too fragmented, needs more than " << kMaxExtents << " extents."; - } - extents->clear(); - } - - return last_extent_seen; -} - -static bool ReadFibmap(int file_fd, const std::string& file_path, - std::vector* extents) { - struct stat s; - if (fstat(file_fd, &s)) { - PLOG(ERROR) << "Failed to stat " << file_path; - return false; - } - - unsigned int blksize; - if (ioctl(file_fd, FIGETBSZ, &blksize) < 0) { - PLOG(ERROR) << "Failed to get FIGETBSZ for " << file_path; - return false; - } - if (!blksize) { - LOG(ERROR) << "Invalid filesystem block size: " << blksize; - return false; - } - - uint64_t num_blocks = (s.st_size + blksize - 1) / blksize; - if (num_blocks > std::numeric_limits::max()) { - LOG(ERROR) << "Too many blocks for FIBMAP (" << num_blocks << ")"; - return false; - } - - for (uint32_t last_block, block_number = 0; block_number < num_blocks; block_number++) { - uint32_t block = block_number; - if (ioctl(file_fd, FIBMAP, &block)) { - PLOG(ERROR) << "Failed to get FIBMAP for file " << file_path; - return false; - } - if (!block) { - LOG(ERROR) << "Logical block " << block_number << " is a hole, which is not supported"; - return false; - } - - if (!extents->empty() && block == last_block + 1) { - extents->back().fe_length += blksize; - } else { - extents->push_back(fiemap_extent{.fe_logical = block_number, - .fe_physical = static_cast(block) * blksize, - .fe_length = static_cast(blksize), - .fe_flags = 0}); - } - last_block = block; - } - return true; -} - -FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_size, bool create, - std::function progress) { - // if 'create' is false, open an existing file and do not truncate. - int open_flags = O_RDWR | O_CLOEXEC; - if (create) { - if (access(file_path.c_str(), F_OK) == 0) { - LOG(WARNING) << "File " << file_path << " already exists, truncating"; - } - open_flags |= O_CREAT | O_TRUNC; - } - ::android::base::unique_fd file_fd( - TEMP_FAILURE_RETRY(open(file_path.c_str(), open_flags, S_IRUSR | S_IWUSR))); - if (file_fd < 0) { - PLOG(ERROR) << "Failed to create file at: " << file_path; - return nullptr; - } - - std::string abs_path; - if (!::android::base::Realpath(file_path, &abs_path)) { - PLOG(ERROR) << "Invalid file path: " << file_path; - cleanup(file_path, create); - return nullptr; - } - - std::string bdev_path; - if (!GetBlockDeviceForFile(abs_path, &bdev_path)) { - LOG(ERROR) << "Failed to get block dev path for file: " << file_path; - cleanup(abs_path, create); - return nullptr; - } - - ::android::base::unique_fd bdev_fd( - TEMP_FAILURE_RETRY(open(bdev_path.c_str(), O_RDONLY | O_CLOEXEC))); - if (bdev_fd < 0) { - PLOG(ERROR) << "Failed to open block device: " << bdev_path; - cleanup(file_path, create); - return nullptr; - } - - uint64_t bdevsz; - if (!GetBlockDeviceSize(bdev_fd, bdev_path, &bdevsz)) { - LOG(ERROR) << "Failed to get block device size for : " << bdev_path; - cleanup(file_path, create); - return nullptr; - } - - if (!create) { - file_size = GetFileSize(abs_path); - if (file_size == 0) { - LOG(ERROR) << "Invalid file size of zero bytes for file: " << abs_path; - return nullptr; - } - } - - uint64_t blocksz; - uint32_t fs_type; - if (!PerformFileChecks(abs_path, file_size, &blocksz, &fs_type)) { - LOG(ERROR) << "Failed to validate file or file system for file:" << abs_path; - cleanup(abs_path, create); - return nullptr; - } - - // Align up to the nearest block size. - if (file_size % blocksz) { - file_size += blocksz - (file_size % blocksz); - } - - if (create) { - if (!AllocateFile(file_fd, abs_path, blocksz, file_size, fs_type, std::move(progress))) { - LOG(ERROR) << "Failed to allocate file: " << abs_path << " of size: " << file_size - << " bytes"; - cleanup(abs_path, create); - return nullptr; - } - } - - // f2fs may move the file blocks around. - if (!PinFile(file_fd, abs_path, fs_type)) { - cleanup(abs_path, create); - LOG(ERROR) << "Failed to pin the file in storage"; - return nullptr; - } - - // now allocate the FiemapWriter and start setting it up - FiemapUniquePtr fmap(new FiemapWriter()); - switch (fs_type) { - case EXT4_SUPER_MAGIC: - case F2FS_SUPER_MAGIC: - if (!ReadFiemap(file_fd, abs_path, &fmap->extents_)) { - LOG(ERROR) << "Failed to read fiemap of file: " << abs_path; - cleanup(abs_path, create); - return nullptr; - } - break; - case MSDOS_SUPER_MAGIC: - if (!ReadFibmap(file_fd, abs_path, &fmap->extents_)) { - LOG(ERROR) << "Failed to read fibmap of file: " << abs_path; - cleanup(abs_path, create); - return nullptr; - } - break; - } - - fmap->file_path_ = abs_path; - fmap->bdev_path_ = bdev_path; - fmap->file_size_ = file_size; - fmap->bdev_size_ = bdevsz; - fmap->fs_type_ = fs_type; - fmap->block_size_ = blocksz; - - LOG(VERBOSE) << "Successfully created FiemapWriter for file " << abs_path << " on block device " - << bdev_path; - return fmap; -} - -} // namespace fiemap_writer -} // namespace android diff --git a/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp b/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp deleted file mode 100644 index dda7dfda4..000000000 --- a/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp +++ /dev/null @@ -1,541 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utility.h" - -namespace android { -namespace fiemap_writer { - -using namespace std; -using namespace std::string_literals; -using namespace android::fiemap_writer; -using unique_fd = android::base::unique_fd; -using LoopDevice = android::dm::LoopDevice; - -std::string gTestDir; -uint64_t testfile_size = 536870912; // default of 512MiB -size_t gBlockSize = 0; - -class FiemapWriterTest : public ::testing::Test { - protected: - void SetUp() override { - const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info(); - testfile = gTestDir + "/"s + tinfo->name(); - } - - void TearDown() override { unlink(testfile.c_str()); } - - // name of the file we use for testing - std::string testfile; -}; - -class SplitFiemapTest : public ::testing::Test { - protected: - void SetUp() override { - const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info(); - testfile = gTestDir + "/"s + tinfo->name(); - } - - void TearDown() override { - std::string message; - if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) { - cerr << "Could not remove all split files: " << message; - } - } - - // name of the file we use for testing - std::string testfile; -}; - -TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) { - // Try creating a file of size ~100TB but aligned to - // 512 byte to make sure block alignment tests don't - // fail. - FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1099511627997184); - EXPECT_EQ(fptr, nullptr); - EXPECT_EQ(access(testfile.c_str(), F_OK), -1); - EXPECT_EQ(errno, ENOENT); -} - -TEST_F(FiemapWriterTest, CreateUnalignedFile) { - // Try creating a file of size 4097 bytes which is guaranteed - // to be unaligned to all known block sizes. - FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize + 1); - ASSERT_NE(fptr, nullptr); - ASSERT_EQ(fptr->size(), gBlockSize * 2); -} - -TEST_F(FiemapWriterTest, CheckFilePath) { - FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize); - ASSERT_NE(fptr, nullptr); - EXPECT_EQ(fptr->size(), gBlockSize); - EXPECT_EQ(fptr->file_path(), testfile); - EXPECT_EQ(access(testfile.c_str(), F_OK), 0); -} - -TEST_F(FiemapWriterTest, CheckFileSize) { - // Create a large-ish file and test that the expected size matches. - FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1024 * 1024 * 16); - ASSERT_NE(fptr, nullptr); - - struct stat s; - ASSERT_EQ(stat(testfile.c_str(), &s), 0); - EXPECT_EQ(static_cast(s.st_size), fptr->size()); -} - -TEST_F(FiemapWriterTest, CheckProgress) { - std::vector expected; - size_t invocations = 0; - auto callback = [&](uint64_t done, uint64_t total) -> bool { - if (invocations >= expected.size()) { - return false; - } - EXPECT_EQ(done, expected[invocations]); - EXPECT_EQ(total, gBlockSize); - invocations++; - return true; - }; - - expected.push_back(gBlockSize); - - auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback)); - EXPECT_NE(ptr, nullptr); - EXPECT_EQ(invocations, expected.size()); -} - -TEST_F(FiemapWriterTest, CheckPinning) { - auto ptr = FiemapWriter::Open(testfile, 4096); - ASSERT_NE(ptr, nullptr); - EXPECT_TRUE(FiemapWriter::HasPinnedExtents(testfile)); -} - -TEST_F(FiemapWriterTest, CheckBlockDevicePath) { - FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize); - EXPECT_EQ(fptr->size(), gBlockSize); - EXPECT_EQ(fptr->bdev_path().find("/dev/block/"), size_t(0)); - EXPECT_EQ(fptr->bdev_path().find("/dev/block/dm-"), string::npos); -} - -TEST_F(FiemapWriterTest, CheckFileCreated) { - FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 32768); - ASSERT_NE(fptr, nullptr); - unique_fd fd(open(testfile.c_str(), O_RDONLY)); - EXPECT_GT(fd, -1); -} - -TEST_F(FiemapWriterTest, CheckFileSizeActual) { - FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size); - ASSERT_NE(fptr, nullptr); - - struct stat sb; - ASSERT_EQ(stat(testfile.c_str(), &sb), 0); - EXPECT_GE(sb.st_size, testfile_size); -} - -TEST_F(FiemapWriterTest, CheckFileExtents) { - FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size); - ASSERT_NE(fptr, nullptr); - EXPECT_GT(fptr->extents().size(), 0); -} - -TEST_F(FiemapWriterTest, ExistingFile) { - // Create the file. - { ASSERT_NE(FiemapWriter::Open(testfile, gBlockSize), nullptr); } - // Test that we can still open it. - { - auto ptr = FiemapWriter::Open(testfile, 0, false); - ASSERT_NE(ptr, nullptr); - EXPECT_GT(ptr->extents().size(), 0); - } -} - -TEST_F(FiemapWriterTest, FileDeletedOnError) { - auto callback = [](uint64_t, uint64_t) -> bool { return false; }; - auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback)); - EXPECT_EQ(ptr, nullptr); - EXPECT_EQ(access(testfile.c_str(), F_OK), -1); - EXPECT_EQ(errno, ENOENT); -} - -TEST_F(FiemapWriterTest, MaxBlockSize) { - ASSERT_GT(DetermineMaximumFileSize(testfile), 0); -} - -TEST_F(FiemapWriterTest, FibmapBlockAddressing) { - FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize); - ASSERT_NE(fptr, nullptr); - - switch (fptr->fs_type()) { - case F2FS_SUPER_MAGIC: - case EXT4_SUPER_MAGIC: - // Skip the test for FIEMAP supported filesystems. This is really - // because f2fs/ext4 have caches that seem to defeat reading back - // directly from the block device, and writing directly is too - // dangerous. - std::cout << "Skipping test, filesystem does not use FIBMAP\n"; - return; - } - - bool uses_dm; - std::string bdev_path; - ASSERT_TRUE(FiemapWriter::GetBlockDeviceForFile(testfile, &bdev_path, &uses_dm)); - - if (uses_dm) { - // We could use a device-mapper wrapper here to bypass encryption, but - // really this test is for FIBMAP correctness on VFAT (where encryption - // is never used), so we don't bother. - std::cout << "Skipping test, block device is metadata encrypted\n"; - return; - } - - std::string data(fptr->size(), '\0'); - for (size_t i = 0; i < data.size(); i++) { - data[i] = 'A' + static_cast(data.size() % 26); - } - - { - unique_fd fd(open(testfile.c_str(), O_WRONLY | O_CLOEXEC)); - ASSERT_GE(fd, 0); - ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size())); - ASSERT_EQ(fsync(fd), 0); - } - - ASSERT_FALSE(fptr->extents().empty()); - const auto& first_extent = fptr->extents()[0]; - - unique_fd bdev(open(fptr->bdev_path().c_str(), O_RDONLY | O_CLOEXEC)); - ASSERT_GE(bdev, 0); - - off_t where = first_extent.fe_physical; - ASSERT_EQ(lseek(bdev, where, SEEK_SET), where); - - // Note: this will fail on encrypted folders. - std::string actual(data.size(), '\0'); - ASSERT_GE(first_extent.fe_length, data.size()); - ASSERT_TRUE(android::base::ReadFully(bdev, actual.data(), actual.size())); - EXPECT_EQ(memcmp(actual.data(), data.data(), data.size()), 0); -} - -TEST_F(SplitFiemapTest, Create) { - auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32); - ASSERT_NE(ptr, nullptr); - - auto extents = ptr->extents(); - - // Destroy the fiemap, closing file handles. This should not delete them. - ptr = nullptr; - - std::vector files; - ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files)); - for (const auto& path : files) { - EXPECT_EQ(access(path.c_str(), F_OK), 0); - } - - ASSERT_GE(extents.size(), files.size()); -} - -TEST_F(SplitFiemapTest, Open) { - { - auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32); - ASSERT_NE(ptr, nullptr); - } - - auto ptr = SplitFiemap::Open(testfile); - ASSERT_NE(ptr, nullptr); - - auto extents = ptr->extents(); - ASSERT_GE(extents.size(), 24); -} - -TEST_F(SplitFiemapTest, DeleteOnFail) { - auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 100, 1); - ASSERT_EQ(ptr, nullptr); - - std::string first_file = testfile + ".0001"; - ASSERT_NE(access(first_file.c_str(), F_OK), 0); - ASSERT_EQ(errno, ENOENT); - ASSERT_NE(access(testfile.c_str(), F_OK), 0); - ASSERT_EQ(errno, ENOENT); -} - -static string ReadSplitFiles(const std::string& base_path, size_t num_files) { - std::string result; - for (int i = 0; i < num_files; i++) { - std::string path = base_path + android::base::StringPrintf(".%04d", i); - std::string data; - if (!android::base::ReadFileToString(path, &data)) { - return {}; - } - result += data; - } - return result; -} - -TEST_F(SplitFiemapTest, WriteWholeFile) { - static constexpr size_t kChunkSize = 32768; - static constexpr size_t kSize = kChunkSize * 3; - auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); - ASSERT_NE(ptr, nullptr); - - auto buffer = std::make_unique(kSize / sizeof(int)); - for (size_t i = 0; i < kSize / sizeof(int); i++) { - buffer[i] = i; - } - ASSERT_TRUE(ptr->Write(buffer.get(), kSize)); - - std::string expected(reinterpret_cast(buffer.get()), kSize); - auto actual = ReadSplitFiles(testfile, 3); - ASSERT_EQ(expected.size(), actual.size()); - EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0); -} - -TEST_F(SplitFiemapTest, WriteFileInChunks1) { - static constexpr size_t kChunkSize = 32768; - static constexpr size_t kSize = kChunkSize * 3; - auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); - ASSERT_NE(ptr, nullptr); - - auto buffer = std::make_unique(kSize / sizeof(int)); - for (size_t i = 0; i < kSize / sizeof(int); i++) { - buffer[i] = i; - } - - // Write in chunks of 1000 (so some writes straddle the boundary of two - // files). - size_t bytes_written = 0; - while (bytes_written < kSize) { - size_t to_write = std::min(kSize - bytes_written, (size_t)1000); - char* data = reinterpret_cast(buffer.get()) + bytes_written; - ASSERT_TRUE(ptr->Write(data, to_write)); - bytes_written += to_write; - } - - std::string expected(reinterpret_cast(buffer.get()), kSize); - auto actual = ReadSplitFiles(testfile, 3); - ASSERT_EQ(expected.size(), actual.size()); - EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0); -} - -TEST_F(SplitFiemapTest, WriteFileInChunks2) { - static constexpr size_t kChunkSize = 32768; - static constexpr size_t kSize = kChunkSize * 3; - auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); - ASSERT_NE(ptr, nullptr); - - auto buffer = std::make_unique(kSize / sizeof(int)); - for (size_t i = 0; i < kSize / sizeof(int); i++) { - buffer[i] = i; - } - - // Write in chunks of 32KiB so every write is exactly at the end of the - // current file. - size_t bytes_written = 0; - while (bytes_written < kSize) { - size_t to_write = std::min(kSize - bytes_written, kChunkSize); - char* data = reinterpret_cast(buffer.get()) + bytes_written; - ASSERT_TRUE(ptr->Write(data, to_write)); - bytes_written += to_write; - } - - std::string expected(reinterpret_cast(buffer.get()), kSize); - auto actual = ReadSplitFiles(testfile, 3); - ASSERT_EQ(expected.size(), actual.size()); - EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0); -} - -TEST_F(SplitFiemapTest, WritePastEnd) { - static constexpr size_t kChunkSize = 32768; - static constexpr size_t kSize = kChunkSize * 3; - auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); - ASSERT_NE(ptr, nullptr); - - auto buffer = std::make_unique(kSize / sizeof(int)); - for (size_t i = 0; i < kSize / sizeof(int); i++) { - buffer[i] = i; - } - ASSERT_TRUE(ptr->Write(buffer.get(), kSize)); - ASSERT_FALSE(ptr->Write(buffer.get(), kSize)); -} - -class VerifyBlockWritesExt4 : public ::testing::Test { - // 2GB Filesystem and 4k block size by default - static constexpr uint64_t block_size = 4096; - static constexpr uint64_t fs_size = 2147483648; - - protected: - void SetUp() override { - fs_path = std::string(getenv("TMPDIR")) + "/ext4_2G.img"; - uint64_t count = fs_size / block_size; - std::string dd_cmd = - ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64 - " count=%" PRIu64 " > /dev/null 2>&1", - fs_path.c_str(), block_size, count); - std::string mkfs_cmd = - ::android::base::StringPrintf("/system/bin/mkfs.ext4 -q %s", fs_path.c_str()); - // create mount point - mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt"; - ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0); - // create file for the file system - int ret = system(dd_cmd.c_str()); - ASSERT_EQ(ret, 0); - // Get and attach a loop device to the filesystem we created - LoopDevice loop_dev(fs_path); - ASSERT_TRUE(loop_dev.valid()); - // create file system - ret = system(mkfs_cmd.c_str()); - ASSERT_EQ(ret, 0); - - // mount the file system - ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "ext4", 0, nullptr), 0); - } - - void TearDown() override { - umount(mntpoint.c_str()); - rmdir(mntpoint.c_str()); - unlink(fs_path.c_str()); - } - - std::string mntpoint; - std::string fs_path; -}; - -class VerifyBlockWritesF2fs : public ::testing::Test { - // 2GB Filesystem and 4k block size by default - static constexpr uint64_t block_size = 4096; - static constexpr uint64_t fs_size = 2147483648; - - protected: - void SetUp() override { - fs_path = std::string(getenv("TMPDIR")) + "/f2fs_2G.img"; - uint64_t count = fs_size / block_size; - std::string dd_cmd = - ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64 - " count=%" PRIu64 " > /dev/null 2>&1", - fs_path.c_str(), block_size, count); - std::string mkfs_cmd = - ::android::base::StringPrintf("/system/bin/make_f2fs -q %s", fs_path.c_str()); - // create mount point - mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt"; - ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0); - // create file for the file system - int ret = system(dd_cmd.c_str()); - ASSERT_EQ(ret, 0); - // Get and attach a loop device to the filesystem we created - LoopDevice loop_dev(fs_path); - ASSERT_TRUE(loop_dev.valid()); - // create file system - ret = system(mkfs_cmd.c_str()); - ASSERT_EQ(ret, 0); - - // mount the file system - ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "f2fs", 0, nullptr), 0); - } - - void TearDown() override { - umount(mntpoint.c_str()); - rmdir(mntpoint.c_str()); - unlink(fs_path.c_str()); - } - - std::string mntpoint; - std::string fs_path; -}; - -bool DetermineBlockSize() { - struct statfs s; - if (statfs(gTestDir.c_str(), &s)) { - std::cerr << "Could not call statfs: " << strerror(errno) << "\n"; - return false; - } - if (!s.f_bsize) { - std::cerr << "Invalid block size: " << s.f_bsize << "\n"; - return false; - } - - gBlockSize = s.f_bsize; - return true; -} - -} // namespace fiemap_writer -} // namespace android - -using namespace android::fiemap_writer; - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - if (argc > 1 && argv[1] == "-h"s) { - cerr << "Usage: [test_dir] [file_size]\n"; - cerr << "\n"; - cerr << "Note: test_dir must be a writable, unencrypted directory.\n"; - exit(EXIT_FAILURE); - } - ::android::base::InitLogging(argv, ::android::base::StderrLogger); - - std::string root_dir = "/data/local/unencrypted"; - if (access(root_dir.c_str(), F_OK)) { - root_dir = "/data"; - } - - std::string tempdir = root_dir + "/XXXXXX"s; - if (!mkdtemp(tempdir.data())) { - cerr << "unable to create tempdir on " << root_dir << "\n"; - exit(EXIT_FAILURE); - } - if (!android::base::Realpath(tempdir, &gTestDir)) { - cerr << "unable to find realpath for " << tempdir; - exit(EXIT_FAILURE); - } - - if (argc > 2) { - testfile_size = strtoull(argv[2], NULL, 0); - if (testfile_size == ULLONG_MAX) { - testfile_size = 512 * 1024 * 1024; - } - } - - if (!DetermineBlockSize()) { - exit(EXIT_FAILURE); - } - - auto result = RUN_ALL_TESTS(); - - std::string cmd = "rm -rf " + gTestDir; - system(cmd.c_str()); - - return result; -} diff --git a/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h b/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h deleted file mode 100644 index ee79262a7..000000000 --- a/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include - -#include -#include -#include - -#include - -namespace android { -namespace fiemap_writer { - -class FiemapWriter; -using FiemapUniquePtr = std::unique_ptr; - -class FiemapWriter final { - public: - // Factory method for FiemapWriter. - // The method returns FiemapUniquePtr that contains all the data necessary to be able to write - // to the given file directly using raw block i/o. The optional progress callback will be - // invoked, if create is true, while the file is being initialized. It receives the bytes - // written and the number of total bytes. If the callback returns false, the operation will - // fail. - // - // Note: when create is true, the file size will be aligned up to the nearest file system - // block. - static FiemapUniquePtr Open(const std::string& file_path, uint64_t file_size, - bool create = true, - std::function progress = {}); - - // Check that a file still has the same extents since it was last opened with FiemapWriter, - // assuming the file was not resized outside of FiemapWriter. Returns false either on error - // or if the file was not pinned. - // - // This will always return true on Ext4. On F2FS, it will return true if either of the - // following cases are true: - // - The file was never pinned. - // - The file is pinned and has not been moved by the GC. - // Thus, this method should only be called for pinned files (such as those returned by - // FiemapWriter::Open). - static bool HasPinnedExtents(const std::string& file_path); - - // Returns the underlying block device of a file. This will look past device-mapper layers - // as long as each layer would not change block mappings (i.e., dm-crypt, dm-bow, and dm- - // default-key tables are okay; dm-linear is not). If a mapping such as dm-linear is found, - // it will be returned in place of any physical block device. - // - // It is the caller's responsibility to check whether the returned block device is acceptable. - // Gsid, for example, will only accept /dev/block/by-name/userdata as the bottom device. - // Callers can check the device name (dm- or loop prefix), inspect sysfs, or compare the major - // number against a boot device. - // - // If device-mapper nodes were encountered, then |uses_dm| will be set to true. - static bool GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path, - bool* uses_dm = nullptr); - - ~FiemapWriter() = default; - - const std::string& file_path() const { return file_path_; }; - uint64_t size() const { return file_size_; }; - const std::string& bdev_path() const { return bdev_path_; }; - uint64_t block_size() const { return block_size_; }; - const std::vector& extents() { return extents_; }; - uint32_t fs_type() const { return fs_type_; } - - // Non-copyable & Non-movable - FiemapWriter(const FiemapWriter&) = delete; - FiemapWriter& operator=(const FiemapWriter&) = delete; - FiemapWriter& operator=(FiemapWriter&&) = delete; - FiemapWriter(FiemapWriter&&) = delete; - - private: - // Name of the file managed by this class. - std::string file_path_; - // Block device on which we have created the file. - std::string bdev_path_; - - // Size in bytes of the file this class is writing - uint64_t file_size_; - - // total size in bytes of the block device - uint64_t bdev_size_; - - // Filesystem type where the file is being created. - // See: for filesystem magic numbers - uint32_t fs_type_; - - // block size as reported by the kernel of the underlying block device; - uint64_t block_size_; - - // This file's fiemap - std::vector extents_; - - FiemapWriter() = default; -}; - -} // namespace fiemap_writer -} // namespace android diff --git a/fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h b/fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h deleted file mode 100644 index 7b977e1bd..000000000 --- a/fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include -#include -#include - -#include - -#include "fiemap_writer.h" - -namespace android { -namespace fiemap_writer { - -// Wrapper around FiemapWriter that is able to split images across files if -// necessary. -class SplitFiemap final { - public: - using ProgressCallback = std::function; - - // Create a new split fiemap file. If |max_piece_size| is 0, the number of - // pieces will be determined automatically by detecting the filesystem. - // Otherwise, the file will be split evenly (with the remainder in the - // final file). - static std::unique_ptr Create(const std::string& file_path, uint64_t file_size, - uint64_t max_piece_size, - ProgressCallback progress = {}); - - // Open an existing split fiemap file. - static std::unique_ptr Open(const std::string& file_path); - - ~SplitFiemap(); - - // Return a list of all files created for a split file. - static bool GetSplitFileList(const std::string& file_path, std::vector* list); - - // Destroy all components of a split file. If the root file does not exist, - // this returns true and does not report an error. - static bool RemoveSplitFiles(const std::string& file_path, std::string* message = nullptr); - - // Return whether all components of a split file still have pinned extents. - bool HasPinnedExtents() const; - - // Helper method for writing data that spans files. Note there is no seek - // method (yet); this starts at 0 and increments the position by |bytes|. - bool Write(const void* data, uint64_t bytes); - - // Flush all writes to all split files. - bool Flush(); - - const std::vector& extents(); - uint32_t block_size() const; - uint64_t size() const { return total_size_; } - const std::string& bdev_path() const; - - // Non-copyable & Non-movable - SplitFiemap(const SplitFiemap&) = delete; - SplitFiemap& operator=(const SplitFiemap&) = delete; - SplitFiemap& operator=(SplitFiemap&&) = delete; - SplitFiemap(SplitFiemap&&) = delete; - - private: - SplitFiemap() = default; - void AddFile(FiemapUniquePtr&& file); - - bool creating_ = false; - std::string list_file_; - std::vector files_; - std::vector extents_; - uint64_t total_size_ = 0; - - // Most recently open file and position for Write(). - size_t cursor_index_ = 0; - uint64_t cursor_file_pos_ = 0; - android::base::unique_fd cursor_fd_; -}; - -} // namespace fiemap_writer -} // namespace android diff --git a/fs_mgr/libfiemap_writer/split_fiemap_writer.cpp b/fs_mgr/libfiemap_writer/split_fiemap_writer.cpp deleted file mode 100644 index 16a82d287..000000000 --- a/fs_mgr/libfiemap_writer/split_fiemap_writer.cpp +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "utility.h" - -namespace android { -namespace fiemap_writer { - -using android::base::unique_fd; - -// We use a four-digit suffix at the end of filenames. -static const size_t kMaxFilePieces = 500; - -std::unique_ptr SplitFiemap::Create(const std::string& file_path, uint64_t file_size, - uint64_t max_piece_size, - ProgressCallback progress) { - if (!file_size) { - LOG(ERROR) << "Cannot create a fiemap for a 0-length file: " << file_path; - return nullptr; - } - - if (!max_piece_size) { - max_piece_size = DetermineMaximumFileSize(file_path); - if (!max_piece_size) { - LOG(ERROR) << "Could not determine maximum file size for " << file_path; - return nullptr; - } - } - - // Call |progress| only when the total percentage would significantly change. - int permille = -1; - uint64_t total_bytes_written = 0; - auto on_progress = [&](uint64_t written, uint64_t) -> bool { - uint64_t actual_written = total_bytes_written + written; - int new_permille = (actual_written * 1000) / file_size; - if (new_permille != permille && actual_written < file_size) { - if (progress && !progress(actual_written, file_size)) { - return false; - } - permille = new_permille; - } - return true; - }; - - std::unique_ptr out(new SplitFiemap()); - out->creating_ = true; - out->list_file_ = file_path; - - // Create the split files. - uint64_t remaining_bytes = file_size; - while (remaining_bytes) { - if (out->files_.size() >= kMaxFilePieces) { - LOG(ERROR) << "Requested size " << file_size << " created too many split files"; - return nullptr; - } - std::string chunk_path = - android::base::StringPrintf("%s.%04d", file_path.c_str(), (int)out->files_.size()); - uint64_t chunk_size = std::min(max_piece_size, remaining_bytes); - auto writer = FiemapWriter::Open(chunk_path, chunk_size, true, on_progress); - if (!writer) { - return nullptr; - } - - // To make sure the alignment doesn't create too much inconsistency, we - // account the *actual* size, not the requested size. - total_bytes_written += writer->size(); - // writer->size() is block size aligned and could be bigger than remaining_bytes - // If remaining_bytes is bigger, set remaining_bytes to 0 to avoid underflow error. - remaining_bytes = remaining_bytes > writer->size() ? (remaining_bytes - writer->size()) : 0; - out->AddFile(std::move(writer)); - } - - // Create the split file list. - unique_fd fd(open(out->list_file_.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0660)); - if (fd < 0) { - PLOG(ERROR) << "Failed to open " << file_path; - return nullptr; - } - - for (const auto& writer : out->files_) { - std::string line = android::base::Basename(writer->file_path()) + "\n"; - if (!android::base::WriteFully(fd, line.data(), line.size())) { - PLOG(ERROR) << "Write failed " << file_path; - return nullptr; - } - } - - // Unset this bit, so we don't unlink on destruction. - out->creating_ = false; - return out; -} - -std::unique_ptr SplitFiemap::Open(const std::string& file_path) { - std::vector files; - if (!GetSplitFileList(file_path, &files)) { - return nullptr; - } - - std::unique_ptr out(new SplitFiemap()); - out->list_file_ = file_path; - - for (const auto& file : files) { - auto writer = FiemapWriter::Open(file, 0, false); - if (!writer) { - // Error was logged in Open(). - return nullptr; - } - out->AddFile(std::move(writer)); - } - return out; -} - -bool SplitFiemap::GetSplitFileList(const std::string& file_path, std::vector* list) { - // This is not the most efficient thing, but it is simple and recovering - // the fiemap/fibmap is much more expensive. - std::string contents; - if (!android::base::ReadFileToString(file_path, &contents, true)) { - PLOG(ERROR) << "Error reading file: " << file_path; - return false; - } - - std::vector names = android::base::Split(contents, "\n"); - std::string dir = android::base::Dirname(file_path); - for (const auto& name : names) { - if (!name.empty()) { - list->emplace_back(dir + "/" + name); - } - } - return true; -} - -bool SplitFiemap::RemoveSplitFiles(const std::string& file_path, std::string* message) { - // Early exit if this does not exist, and do not report an error. - if (access(file_path.c_str(), F_OK) && errno == ENOENT) { - return true; - } - - bool ok = true; - std::vector files; - if (GetSplitFileList(file_path, &files)) { - for (const auto& file : files) { - ok &= android::base::RemoveFileIfExists(file, message); - } - } - ok &= android::base::RemoveFileIfExists(file_path, message); - return ok; -} - -bool SplitFiemap::HasPinnedExtents() const { - for (const auto& file : files_) { - if (!FiemapWriter::HasPinnedExtents(file->file_path())) { - return false; - } - } - return true; -} - -const std::vector& SplitFiemap::extents() { - if (extents_.empty()) { - for (const auto& file : files_) { - const auto& extents = file->extents(); - extents_.insert(extents_.end(), extents.begin(), extents.end()); - } - } - return extents_; -} - -bool SplitFiemap::Write(const void* data, uint64_t bytes) { - // Open the current file. - FiemapWriter* file = files_[cursor_index_].get(); - - const uint8_t* data_ptr = reinterpret_cast(data); - uint64_t bytes_remaining = bytes; - while (bytes_remaining) { - // How many bytes can we write into the current file? - uint64_t file_bytes_left = file->size() - cursor_file_pos_; - if (!file_bytes_left) { - if (cursor_index_ == files_.size() - 1) { - LOG(ERROR) << "write past end of file requested"; - return false; - } - - // No space left in the current file, but we have more files to - // use, so prep the next one. - cursor_fd_ = {}; - cursor_file_pos_ = 0; - file = files_[++cursor_index_].get(); - file_bytes_left = file->size(); - } - - // Open the current file if it's not open. - if (cursor_fd_ < 0) { - cursor_fd_.reset(open(file->file_path().c_str(), O_CLOEXEC | O_WRONLY)); - if (cursor_fd_ < 0) { - PLOG(ERROR) << "open failed: " << file->file_path(); - return false; - } - CHECK(cursor_file_pos_ == 0); - } - - if (!FiemapWriter::HasPinnedExtents(file->file_path())) { - LOG(ERROR) << "file is no longer pinned: " << file->file_path(); - return false; - } - - uint64_t bytes_to_write = std::min(file_bytes_left, bytes_remaining); - if (!android::base::WriteFully(cursor_fd_, data_ptr, bytes_to_write)) { - PLOG(ERROR) << "write failed: " << file->file_path(); - return false; - } - data_ptr += bytes_to_write; - bytes_remaining -= bytes_to_write; - cursor_file_pos_ += bytes_to_write; - } - - // If we've reached the end of the current file, close it for sanity. - if (cursor_file_pos_ == file->size()) { - cursor_fd_ = {}; - } - return true; -} - -bool SplitFiemap::Flush() { - for (const auto& file : files_) { - unique_fd fd(open(file->file_path().c_str(), O_RDONLY | O_CLOEXEC)); - if (fd < 0) { - PLOG(ERROR) << "open failed: " << file->file_path(); - return false; - } - if (fsync(fd)) { - PLOG(ERROR) << "fsync failed: " << file->file_path(); - return false; - } - } - return true; -} - -SplitFiemap::~SplitFiemap() { - if (!creating_) { - return; - } - - // We failed to finish creating, so unlink everything. - unlink(list_file_.c_str()); - for (auto&& file : files_) { - std::string path = file->file_path(); - file = nullptr; - - unlink(path.c_str()); - } -} - -void SplitFiemap::AddFile(FiemapUniquePtr&& file) { - total_size_ += file->size(); - files_.emplace_back(std::move(file)); -} - -uint32_t SplitFiemap::block_size() const { - return files_[0]->block_size(); -} - -const std::string& SplitFiemap::bdev_path() const { - return files_[0]->bdev_path(); -} - -} // namespace fiemap_writer -} // namespace android diff --git a/fs_mgr/libfiemap_writer/testdata/file_32k b/fs_mgr/libfiemap_writer/testdata/file_32k deleted file mode 100644 index 12f3be4dd..000000000 Binary files a/fs_mgr/libfiemap_writer/testdata/file_32k and /dev/null differ diff --git a/fs_mgr/libfiemap_writer/testdata/file_4k b/fs_mgr/libfiemap_writer/testdata/file_4k deleted file mode 100644 index 08e7df176..000000000 Binary files a/fs_mgr/libfiemap_writer/testdata/file_4k and /dev/null differ diff --git a/fs_mgr/libfiemap_writer/testdata/unaligned_file b/fs_mgr/libfiemap_writer/testdata/unaligned_file deleted file mode 100644 index c107c2637..000000000 Binary files a/fs_mgr/libfiemap_writer/testdata/unaligned_file and /dev/null differ diff --git a/fs_mgr/libfiemap_writer/utility.cpp b/fs_mgr/libfiemap_writer/utility.cpp deleted file mode 100644 index 192ec1602..000000000 --- a/fs_mgr/libfiemap_writer/utility.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "utility.h" - -#include -#include -#include - -#include -#include - -namespace android { -namespace fiemap_writer { - -uint64_t DetermineMaximumFileSize(const std::string& file_path) { - // Create the smallest file possible (one block). - auto writer = FiemapWriter::Open(file_path, 1); - if (!writer) { - return 0; - } - - uint64_t result = 0; - switch (writer->fs_type()) { - case EXT4_SUPER_MAGIC: - // The minimum is 16GiB, so just report that. If we wanted we could parse the - // superblock and figure out if 64-bit support is enabled. - result = 17179869184ULL; - break; - case F2FS_SUPER_MAGIC: - // Formula is from https://www.kernel.org/doc/Documentation/filesystems/f2fs.txt - // 4KB * (923 + 2 * 1018 + 2 * 1018 * 1018 + 1018 * 1018 * 1018) := 3.94TB. - result = 4329690886144ULL; - break; - case MSDOS_SUPER_MAGIC: - // 4GB-1, which we want aligned to the block size. - result = 4294967295; - result -= (result % writer->block_size()); - break; - default: - LOG(ERROR) << "Unknown file system type: " << writer->fs_type(); - break; - } - - // Close and delete the temporary file. - writer = nullptr; - unlink(file_path.c_str()); - - return result; -} - -} // namespace fiemap_writer -} // namespace android diff --git a/fs_mgr/libfiemap_writer/utility.h b/fs_mgr/libfiemap_writer/utility.h deleted file mode 100644 index 2d418dae4..000000000 --- a/fs_mgr/libfiemap_writer/utility.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -namespace android { -namespace fiemap_writer { - -// Given a file that will be created, determine the maximum size its containing -// filesystem allows. Note this is a theoretical maximum size; free space is -// ignored entirely. -uint64_t DetermineMaximumFileSize(const std::string& file_path); - -} // namespace fiemap_writer -} // namespace android