diff --git a/fs_mgr/libfiemap_writer/.clang-format b/fs_mgr/libfiemap_writer/.clang-format new file mode 120000 index 000000000..8b770a15c --- /dev/null +++ b/fs_mgr/libfiemap_writer/.clang-format @@ -0,0 +1 @@ +../../.clang-format-4 \ No newline at end of file diff --git a/fs_mgr/libfiemap_writer/Android.bp b/fs_mgr/libfiemap_writer/Android.bp new file mode 100644 index 000000000..ed209aa9c --- /dev/null +++ b/fs_mgr/libfiemap_writer/Android.bp @@ -0,0 +1,64 @@ +// +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +cc_library_static { + name: "libfiemap_writer", + defaults: ["fs_mgr_defaults"], + recovery_available: true, + export_include_dirs: ["include"], + cflags: [ + "-D_FILE_OFFSET_BITS=64", + ], + + srcs: [ + "fiemap_writer.cpp", + "split_fiemap_writer.cpp", + "utility.cpp", + ], + + static_libs: [ + "libdm", + "libext4_utils", + ], + + header_libs: [ + "libbase_headers", + "liblog_headers", + ], +} + +cc_test { + name: "fiemap_writer_test", + cflags: [ + "-D_FILE_OFFSET_BITS=64", + ], + static_libs: [ + "libbase", + "libdm", + "libfiemap_writer", + "liblog", + ], + + data: [ + "testdata/unaligned_file", + "testdata/file_4k", + "testdata/file_32k", + ], + + srcs: [ + "fiemap_writer_test.cpp", + ], +} diff --git a/fs_mgr/libfiemap_writer/Android.mk b/fs_mgr/libfiemap_writer/Android.mk new file mode 100644 index 000000000..3c07b8ea6 --- /dev/null +++ b/fs_mgr/libfiemap_writer/Android.mk @@ -0,0 +1,22 @@ +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +LOCAL_PATH := $(call my-dir) + +include $(CLEAR_VARS) + +LOCAL_MODULE := VtsFiemapWriterTest +-include test/vts/tools/build/Android.host_config.mk diff --git a/fs_mgr/libfiemap_writer/AndroidTest.xml b/fs_mgr/libfiemap_writer/AndroidTest.xml new file mode 100644 index 000000000..08cff0e24 --- /dev/null +++ b/fs_mgr/libfiemap_writer/AndroidTest.xml @@ -0,0 +1,29 @@ + + + + diff --git a/fs_mgr/libfiemap_writer/fiemap_writer.cpp b/fs_mgr/libfiemap_writer/fiemap_writer.cpp new file mode 100644 index 000000000..0a3ba6c24 --- /dev/null +++ b/fs_mgr/libfiemap_writer/fiemap_writer.cpp @@ -0,0 +1,745 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace android { +namespace fiemap_writer { + +using namespace android::dm; + +// We are expecting no more than 512 extents in a fiemap of the file we create. +// If we find more, then it is treated as error for now. +static constexpr const uint32_t kMaxExtents = 512; + +// TODO: Fallback to using fibmap if FIEMAP_EXTENT_MERGED is set. +static constexpr const uint32_t kUnsupportedExtentFlags = + FIEMAP_EXTENT_UNKNOWN | FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_DELALLOC | + FIEMAP_EXTENT_NOT_ALIGNED | FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_DATA_TAIL | + FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_SHARED | FIEMAP_EXTENT_MERGED; + +// Large file support must be enabled. +static_assert(sizeof(off_t) == sizeof(uint64_t)); + +static inline void cleanup(const std::string& file_path, bool created) { + if (created) { + unlink(file_path.c_str()); + } +} + +static bool BlockDeviceToName(uint32_t major, uint32_t minor, std::string* bdev_name) { + // The symlinks in /sys/dev/block point to the block device node under /sys/device/.. + // The directory name in the target corresponds to the name of the block device. We use + // that to extract the block device name. + // e.g for block device name 'ram0', there exists a symlink named '1:0' in /sys/dev/block as + // follows. + // 1:0 -> ../../devices/virtual/block/ram0 + std::string sysfs_path = ::android::base::StringPrintf("/sys/dev/block/%u:%u", major, minor); + std::string sysfs_bdev; + + if (!::android::base::Readlink(sysfs_path, &sysfs_bdev)) { + PLOG(ERROR) << "Failed to read link at: " << sysfs_path; + return false; + } + + *bdev_name = ::android::base::Basename(sysfs_bdev); + // Paranoid sanity check to make sure we just didn't get the + // input in return as-is. + if (sysfs_bdev == *bdev_name) { + LOG(ERROR) << "Malformed symlink for block device: " << sysfs_bdev; + return false; + } + + return true; +} + +static bool ValidateDmTarget(const DeviceMapper::TargetInfo& target) { + const auto& entry = target.spec; + if (entry.sector_start != 0) { + LOG(INFO) << "Stopping at target with non-zero starting sector"; + return false; + } + + auto target_type = DeviceMapper::GetTargetType(entry); + if (target_type == "bow" || target_type == "default-key" || target_type == "crypt") { + return true; + } + if (target_type == "linear") { + auto pieces = android::base::Split(target.data, " "); + if (pieces[1] != "0") { + LOG(INFO) << "Stopping at complex linear target with non-zero starting sector: " + << pieces[1]; + return false; + } + return true; + } + + LOG(INFO) << "Stopping at complex target type " << target_type; + return false; +} + +static bool DeviceMapperStackPop(const std::string& bdev, std::string* bdev_raw) { + *bdev_raw = bdev; + + if (!::android::base::StartsWith(bdev, "dm-")) { + // We are at the bottom of the device mapper stack. + return true; + } + + // Get the device name. + auto dm_name_file = "/sys/block/" + bdev + "/dm/name"; + std::string dm_name; + if (!android::base::ReadFileToString(dm_name_file, &dm_name)) { + PLOG(ERROR) << "Could not read file: " << dm_name_file; + return false; + } + dm_name = android::base::Trim(dm_name); + + auto& dm = DeviceMapper::Instance(); + std::vector table; + if (!dm.GetTableInfo(dm_name, &table)) { + LOG(ERROR) << "Could not read device-mapper table for " << dm_name << " at " << bdev; + return false; + } + + // The purpose of libfiemap_writer is to provide an extent-based view into + // a file. This is difficult if devices are not layered in a 1:1 manner; + // we would have to translate and break up extents based on the actual + // block mapping. Since this is too complex, we simply stop processing + // the device-mapper stack if we encounter a complex case. + // + // It is up to the caller to decide whether stopping at a virtual block + // device is allowable. In most cases it is not, because we want either + // "userdata" or an external volume. It is useful for tests however. + // Callers can check by comparing the device number to that of userdata, + // or by checking whether is a device-mapper node. + if (table.size() > 1) { + LOG(INFO) << "Stopping at complex table for " << dm_name << " at " << bdev; + return true; + } + if (!ValidateDmTarget(table[0])) { + return true; + } + + auto dm_leaf_dir = "/sys/block/" + bdev + "/slaves"; + auto d = std::unique_ptr(opendir(dm_leaf_dir.c_str()), closedir); + if (d == nullptr) { + PLOG(ERROR) << "Failed to open: " << dm_leaf_dir; + return false; + } + + struct dirent* de; + uint32_t num_leaves = 0; + std::string bdev_next = ""; + while ((de = readdir(d.get())) != nullptr) { + if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) { + continue; + } + + // We set the first name we find here + if (bdev_next.empty()) { + bdev_next = de->d_name; + } + num_leaves++; + } + + // if we have more than one leaves, we return immediately. We can't continue to create the + // file since we don't know how to write it out using fiemap, so it will be readable via the + // underlying block devices later. The reader will also have to construct the same device mapper + // target in order read the file out. + if (num_leaves > 1) { + LOG(ERROR) << "Found " << num_leaves << " leaf block devices under device mapper device " + << bdev; + return false; + } + + // recursively call with the block device we found in order to pop the device mapper stack. + return DeviceMapperStackPop(bdev_next, bdev_raw); +} + +bool FiemapWriter::GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path, + bool* uses_dm) { + struct stat sb; + if (stat(file_path.c_str(), &sb)) { + PLOG(ERROR) << "Failed to get stat for: " << file_path; + return false; + } + + std::string bdev; + if (!BlockDeviceToName(major(sb.st_dev), minor(sb.st_dev), &bdev)) { + LOG(ERROR) << "Failed to get block device name for " << major(sb.st_dev) << ":" + << minor(sb.st_dev); + return false; + } + + std::string bdev_raw; + if (!DeviceMapperStackPop(bdev, &bdev_raw)) { + LOG(ERROR) << "Failed to get the bottom of the device mapper stack for device: " << bdev; + return false; + } + + if (uses_dm) { + *uses_dm = (bdev_raw != bdev); + } + + LOG(DEBUG) << "Popped device (" << bdev_raw << ") from device mapper stack starting with (" + << bdev << ")"; + + *bdev_path = ::android::base::StringPrintf("/dev/block/%s", bdev_raw.c_str()); + + // Make sure we are talking to a block device before calling it a success. + if (stat(bdev_path->c_str(), &sb)) { + PLOG(ERROR) << "Failed to get stat for block device: " << *bdev_path; + return false; + } + + if ((sb.st_mode & S_IFMT) != S_IFBLK) { + PLOG(ERROR) << "File: " << *bdev_path << " is not a block device"; + return false; + } + + return true; +} + +static bool GetBlockDeviceSize(int bdev_fd, const std::string& bdev_path, uint64_t* bdev_size) { + uint64_t size_in_bytes = 0; + if (ioctl(bdev_fd, BLKGETSIZE64, &size_in_bytes)) { + PLOG(ERROR) << "Failed to get total size for: " << bdev_path; + return false; + } + + *bdev_size = size_in_bytes; + + return true; +} + +static uint64_t GetFileSize(const std::string& file_path) { + struct stat sb; + if (stat(file_path.c_str(), &sb)) { + PLOG(ERROR) << "Failed to get size for file: " << file_path; + return 0; + } + + return sb.st_size; +} + +static bool PerformFileChecks(const std::string& file_path, uint64_t file_size, uint64_t* blocksz, + uint32_t* fs_type) { + struct statfs64 sfs; + if (statfs64(file_path.c_str(), &sfs)) { + PLOG(ERROR) << "Failed to read file system status at: " << file_path; + return false; + } + + if (!sfs.f_bsize) { + LOG(ERROR) << "Unsupported block size: " << sfs.f_bsize; + return false; + } + + // Check if the filesystem is of supported types. + // Only ext4, f2fs, and vfat are tested and supported. + switch (sfs.f_type) { + case EXT4_SUPER_MAGIC: + case F2FS_SUPER_MAGIC: + case MSDOS_SUPER_MAGIC: + break; + default: + LOG(ERROR) << "Unsupported file system type: 0x" << std::hex << sfs.f_type; + return false; + } + + uint64_t available_bytes = sfs.f_bsize * sfs.f_bavail; + if (available_bytes <= file_size) { + LOG(ERROR) << "Not enough free space in file system to create file of size : " << file_size; + return false; + } + + *blocksz = sfs.f_bsize; + *fs_type = sfs.f_type; + return true; +} + +static bool FallocateFallback(int file_fd, uint64_t block_size, uint64_t file_size, + const std::string& file_path, + const std::function& on_progress) { + // Even though this is much faster than writing zeroes, it is still slow + // enough that we need to fire the progress callback periodically. To + // easily achieve this, we seek in chunks. We use 1000 chunks since + // normally we only fire the callback on 1/1000th increments. + uint64_t bytes_per_chunk = std::max(file_size / 1000, block_size); + + // Seek just to the end of each chunk and write a single byte, causing + // the filesystem to allocate blocks. + off_t cursor = 0; + off_t end = static_cast(file_size); + while (cursor < end) { + cursor = std::min(static_cast(cursor + bytes_per_chunk), end); + auto rv = TEMP_FAILURE_RETRY(lseek(file_fd, cursor - 1, SEEK_SET)); + if (rv < 0) { + PLOG(ERROR) << "Failed to lseek " << file_path; + return false; + } + if (rv != cursor - 1) { + LOG(ERROR) << "Seek returned wrong offset " << rv << " for file " << file_path; + return false; + } + char buffer[] = {0}; + if (!android::base::WriteFully(file_fd, buffer, 1)) { + PLOG(ERROR) << "Write failed: " << file_path; + return false; + } + if (on_progress && !on_progress(cursor, file_size)) { + return false; + } + } + return true; +} + +static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blocksz, + uint64_t file_size, unsigned int fs_type, + std::function on_progress) { + // Reserve space for the file on the file system and write it out to make sure the extents + // don't come back unwritten. Return from this function with the kernel file offset set to 0. + // If the filesystem is f2fs, then we also PIN the file on disk to make sure the blocks + // aren't moved around. + switch (fs_type) { + case EXT4_SUPER_MAGIC: + case F2FS_SUPER_MAGIC: + if (fallocate(file_fd, FALLOC_FL_ZERO_RANGE, 0, file_size)) { + PLOG(ERROR) << "Failed to allocate space for file: " << file_path + << " size: " << file_size; + return false; + } + break; + case MSDOS_SUPER_MAGIC: + // fallocate() is not supported, and not needed, since VFAT does not support holes. + // Instead we can perform a much faster allocation. + return FallocateFallback(file_fd, blocksz, file_size, file_path, on_progress); + default: + LOG(ERROR) << "Missing fallocate() support for file system " << fs_type; + return false; + } + + // write zeroes in 'blocksz' byte increments until we reach file_size to make sure the data + // blocks are actually written to by the file system and thus getting rid of the holes in the + // file. + auto buffer = std::unique_ptr(calloc(1, blocksz), free); + if (buffer == nullptr) { + LOG(ERROR) << "failed to allocate memory for writing file"; + return false; + } + + off64_t offset = lseek64(file_fd, 0, SEEK_SET); + if (offset < 0) { + PLOG(ERROR) << "Failed to seek at the beginning of : " << file_path; + return false; + } + + int permille = -1; + while (offset < file_size) { + if (!::android::base::WriteFully(file_fd, buffer.get(), blocksz)) { + PLOG(ERROR) << "Failed to write" << blocksz << " bytes at offset" << offset + << " in file " << file_path; + return false; + } + + offset += blocksz; + + // Don't invoke the callback every iteration - wait until a significant + // chunk (here, 1/1000th) of the data has been processed. + int new_permille = (static_cast(offset) * 1000) / file_size; + if (new_permille != permille && static_cast(offset) != file_size) { + if (on_progress && !on_progress(offset, file_size)) { + return false; + } + permille = new_permille; + } + } + + if (lseek64(file_fd, 0, SEEK_SET) < 0) { + PLOG(ERROR) << "Failed to reset offset at the beginning of : " << file_path; + return false; + } + + // flush all writes here .. + if (fsync(file_fd)) { + PLOG(ERROR) << "Failed to synchronize written file:" << file_path; + return false; + } + + // Send one last progress notification. + if (on_progress && !on_progress(file_size, file_size)) { + return false; + } + return true; +} + +static bool PinFile(int file_fd, const std::string& file_path, uint32_t fs_type) { + if (fs_type != F2FS_SUPER_MAGIC) { + // No pinning necessary for ext4/msdos. The blocks, once allocated, are + // expected to be fixed. + return true; + } + +// F2FS-specific ioctl +// It requires the below kernel commit merged in v4.16-rc1. +// 1ad71a27124c ("f2fs: add an ioctl to disable GC for specific file") +// In android-4.4, +// 56ee1e817908 ("f2fs: updates on v4.16-rc1") +// In android-4.9, +// 2f17e34672a8 ("f2fs: updates on v4.16-rc1") +// In android-4.14, +// ce767d9a55bc ("f2fs: updates on v4.16-rc1") +#ifndef F2FS_IOC_SET_PIN_FILE +#ifndef F2FS_IOCTL_MAGIC +#define F2FS_IOCTL_MAGIC 0xf5 +#endif +#define F2FS_IOC_SET_PIN_FILE _IOW(F2FS_IOCTL_MAGIC, 13, __u32) +#endif + + uint32_t pin_status = 1; + int error = ioctl(file_fd, F2FS_IOC_SET_PIN_FILE, &pin_status); + if (error < 0) { + if ((errno == ENOTTY) || (errno == ENOTSUP)) { + PLOG(ERROR) << "Failed to pin file, not supported by kernel: " << file_path; + } else { + PLOG(ERROR) << "Failed to pin file: " << file_path; + } + return false; + } + + return true; +} + +static bool IsFilePinned(int file_fd, const std::string& file_path, uint32_t fs_type) { + if (fs_type != F2FS_SUPER_MAGIC) { + // No pinning necessary for ext4 or vfat. The blocks, once allocated, + // are expected to be fixed. + return true; + } + +// F2FS-specific ioctl +// It requires the below kernel commit merged in v4.16-rc1. +// 1ad71a27124c ("f2fs: add an ioctl to disable GC for specific file") +// In android-4.4, +// 56ee1e817908 ("f2fs: updates on v4.16-rc1") +// In android-4.9, +// 2f17e34672a8 ("f2fs: updates on v4.16-rc1") +// In android-4.14, +// ce767d9a55bc ("f2fs: updates on v4.16-rc1") +#ifndef F2FS_IOC_GET_PIN_FILE +#ifndef F2FS_IOCTL_MAGIC +#define F2FS_IOCTL_MAGIC 0xf5 +#endif +#define F2FS_IOC_GET_PIN_FILE _IOR(F2FS_IOCTL_MAGIC, 14, __u32) +#endif + + // f2fs: export FS_NOCOW_FL flag to user + uint32_t flags; + int error = ioctl(file_fd, FS_IOC_GETFLAGS, &flags); + if (error < 0) { + if ((errno == ENOTTY) || (errno == ENOTSUP)) { + PLOG(ERROR) << "Failed to get flags, not supported by kernel: " << file_path; + } else { + PLOG(ERROR) << "Failed to get flags: " << file_path; + } + return false; + } + if (!(flags & FS_NOCOW_FL)) { + LOG(ERROR) << "It is not pinned: " << file_path; + return false; + } + + // F2FS_IOC_GET_PIN_FILE returns the number of blocks moved. + uint32_t moved_blocks_nr; + error = ioctl(file_fd, F2FS_IOC_GET_PIN_FILE, &moved_blocks_nr); + if (error < 0) { + if ((errno == ENOTTY) || (errno == ENOTSUP)) { + PLOG(ERROR) << "Failed to get file pin status, not supported by kernel: " << file_path; + } else { + PLOG(ERROR) << "Failed to get file pin status: " << file_path; + } + return false; + } + + if (moved_blocks_nr) { + LOG(ERROR) << moved_blocks_nr << " blocks moved in file " << file_path; + } + return moved_blocks_nr == 0; +} + +bool FiemapWriter::HasPinnedExtents(const std::string& file_path) { + android::base::unique_fd fd(open(file_path.c_str(), O_NOFOLLOW | O_CLOEXEC | O_RDONLY)); + if (fd < 0) { + PLOG(ERROR) << "open: " << file_path; + return false; + } + + struct statfs64 sfs; + if (fstatfs64(fd, &sfs)) { + PLOG(ERROR) << "fstatfs64: " << file_path; + return false; + } + return IsFilePinned(fd, file_path, sfs.f_type); +} + +static bool ReadFiemap(int file_fd, const std::string& file_path, + std::vector* extents) { + uint64_t fiemap_size = + sizeof(struct fiemap_extent) + kMaxExtents * sizeof(struct fiemap_extent); + auto buffer = std::unique_ptr(calloc(1, fiemap_size), free); + if (buffer == nullptr) { + LOG(ERROR) << "Failed to allocate memory for fiemap"; + return false; + } + + struct fiemap* fiemap = reinterpret_cast(buffer.get()); + fiemap->fm_start = 0; + fiemap->fm_length = UINT64_MAX; + // make sure file is synced to disk before we read the fiemap + fiemap->fm_flags = FIEMAP_FLAG_SYNC; + fiemap->fm_extent_count = kMaxExtents; + + if (ioctl(file_fd, FS_IOC_FIEMAP, fiemap)) { + PLOG(ERROR) << "Failed to get FIEMAP from the kernel for file: " << file_path; + return false; + } + + if (fiemap->fm_mapped_extents == 0) { + LOG(ERROR) << "File " << file_path << " has zero extents"; + return false; + } + + // Iterate through each extent read and make sure its valid before adding it to the vector + bool last_extent_seen = false; + struct fiemap_extent* extent = &fiemap->fm_extents[0]; + for (uint32_t i = 0; i < fiemap->fm_mapped_extents; i++, extent++) { + // LogExtent(i + 1, *extent); + if (extent->fe_flags & kUnsupportedExtentFlags) { + LOG(ERROR) << "Extent " << i + 1 << " of file " << file_path + << " has unsupported flags"; + extents->clear(); + return false; + } + + if (extent->fe_flags & FIEMAP_EXTENT_LAST) { + last_extent_seen = true; + if (i != (fiemap->fm_mapped_extents - 1)) { + LOG(WARNING) << "Extents are being received out-of-order"; + } + } + extents->emplace_back(std::move(*extent)); + } + + if (!last_extent_seen) { + // The file is possibly too fragmented. + if (fiemap->fm_mapped_extents == kMaxExtents) { + LOG(ERROR) << "File is too fragmented, needs more than " << kMaxExtents << " extents."; + } + extents->clear(); + } + + return last_extent_seen; +} + +static bool ReadFibmap(int file_fd, const std::string& file_path, + std::vector* extents) { + struct stat s; + if (fstat(file_fd, &s)) { + PLOG(ERROR) << "Failed to stat " << file_path; + return false; + } + + unsigned int blksize; + if (ioctl(file_fd, FIGETBSZ, &blksize) < 0) { + PLOG(ERROR) << "Failed to get FIGETBSZ for " << file_path; + return false; + } + if (!blksize) { + LOG(ERROR) << "Invalid filesystem block size: " << blksize; + return false; + } + + uint64_t num_blocks = (s.st_size + blksize - 1) / blksize; + if (num_blocks > std::numeric_limits::max()) { + LOG(ERROR) << "Too many blocks for FIBMAP (" << num_blocks << ")"; + return false; + } + + for (uint32_t last_block, block_number = 0; block_number < num_blocks; block_number++) { + uint32_t block = block_number; + if (ioctl(file_fd, FIBMAP, &block)) { + PLOG(ERROR) << "Failed to get FIBMAP for file " << file_path; + return false; + } + if (!block) { + LOG(ERROR) << "Logical block " << block_number << " is a hole, which is not supported"; + return false; + } + + if (!extents->empty() && block == last_block + 1) { + extents->back().fe_length += blksize; + } else { + extents->push_back(fiemap_extent{.fe_logical = block_number, + .fe_physical = static_cast(block) * blksize, + .fe_length = static_cast(blksize), + .fe_flags = 0}); + } + last_block = block; + } + return true; +} + +FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_size, bool create, + std::function progress) { + // if 'create' is false, open an existing file and do not truncate. + int open_flags = O_RDWR | O_CLOEXEC; + if (create) { + if (access(file_path.c_str(), F_OK) == 0) { + LOG(WARNING) << "File " << file_path << " already exists, truncating"; + } + open_flags |= O_CREAT | O_TRUNC; + } + ::android::base::unique_fd file_fd( + TEMP_FAILURE_RETRY(open(file_path.c_str(), open_flags, S_IRUSR | S_IWUSR))); + if (file_fd < 0) { + PLOG(ERROR) << "Failed to create file at: " << file_path; + return nullptr; + } + + std::string abs_path; + if (!::android::base::Realpath(file_path, &abs_path)) { + PLOG(ERROR) << "Invalid file path: " << file_path; + cleanup(file_path, create); + return nullptr; + } + + std::string bdev_path; + if (!GetBlockDeviceForFile(abs_path, &bdev_path)) { + LOG(ERROR) << "Failed to get block dev path for file: " << file_path; + cleanup(abs_path, create); + return nullptr; + } + + ::android::base::unique_fd bdev_fd( + TEMP_FAILURE_RETRY(open(bdev_path.c_str(), O_RDONLY | O_CLOEXEC))); + if (bdev_fd < 0) { + PLOG(ERROR) << "Failed to open block device: " << bdev_path; + cleanup(file_path, create); + return nullptr; + } + + uint64_t bdevsz; + if (!GetBlockDeviceSize(bdev_fd, bdev_path, &bdevsz)) { + LOG(ERROR) << "Failed to get block device size for : " << bdev_path; + cleanup(file_path, create); + return nullptr; + } + + if (!create) { + file_size = GetFileSize(abs_path); + if (file_size == 0) { + LOG(ERROR) << "Invalid file size of zero bytes for file: " << abs_path; + return nullptr; + } + } + + uint64_t blocksz; + uint32_t fs_type; + if (!PerformFileChecks(abs_path, file_size, &blocksz, &fs_type)) { + LOG(ERROR) << "Failed to validate file or file system for file:" << abs_path; + cleanup(abs_path, create); + return nullptr; + } + + // Align up to the nearest block size. + if (file_size % blocksz) { + file_size += blocksz - (file_size % blocksz); + } + + if (create) { + if (!AllocateFile(file_fd, abs_path, blocksz, file_size, fs_type, std::move(progress))) { + LOG(ERROR) << "Failed to allocate file: " << abs_path << " of size: " << file_size + << " bytes"; + cleanup(abs_path, create); + return nullptr; + } + } + + // f2fs may move the file blocks around. + if (!PinFile(file_fd, abs_path, fs_type)) { + cleanup(abs_path, create); + LOG(ERROR) << "Failed to pin the file in storage"; + return nullptr; + } + + // now allocate the FiemapWriter and start setting it up + FiemapUniquePtr fmap(new FiemapWriter()); + switch (fs_type) { + case EXT4_SUPER_MAGIC: + case F2FS_SUPER_MAGIC: + if (!ReadFiemap(file_fd, abs_path, &fmap->extents_)) { + LOG(ERROR) << "Failed to read fiemap of file: " << abs_path; + cleanup(abs_path, create); + return nullptr; + } + break; + case MSDOS_SUPER_MAGIC: + if (!ReadFibmap(file_fd, abs_path, &fmap->extents_)) { + LOG(ERROR) << "Failed to read fibmap of file: " << abs_path; + cleanup(abs_path, create); + return nullptr; + } + break; + } + + fmap->file_path_ = abs_path; + fmap->bdev_path_ = bdev_path; + fmap->file_size_ = file_size; + fmap->bdev_size_ = bdevsz; + fmap->fs_type_ = fs_type; + fmap->block_size_ = blocksz; + + LOG(VERBOSE) << "Successfully created FiemapWriter for file " << abs_path << " on block device " + << bdev_path; + return fmap; +} + +} // namespace fiemap_writer +} // namespace android diff --git a/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp b/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp new file mode 100644 index 000000000..dda7dfda4 --- /dev/null +++ b/fs_mgr/libfiemap_writer/fiemap_writer_test.cpp @@ -0,0 +1,541 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utility.h" + +namespace android { +namespace fiemap_writer { + +using namespace std; +using namespace std::string_literals; +using namespace android::fiemap_writer; +using unique_fd = android::base::unique_fd; +using LoopDevice = android::dm::LoopDevice; + +std::string gTestDir; +uint64_t testfile_size = 536870912; // default of 512MiB +size_t gBlockSize = 0; + +class FiemapWriterTest : public ::testing::Test { + protected: + void SetUp() override { + const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info(); + testfile = gTestDir + "/"s + tinfo->name(); + } + + void TearDown() override { unlink(testfile.c_str()); } + + // name of the file we use for testing + std::string testfile; +}; + +class SplitFiemapTest : public ::testing::Test { + protected: + void SetUp() override { + const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info(); + testfile = gTestDir + "/"s + tinfo->name(); + } + + void TearDown() override { + std::string message; + if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) { + cerr << "Could not remove all split files: " << message; + } + } + + // name of the file we use for testing + std::string testfile; +}; + +TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) { + // Try creating a file of size ~100TB but aligned to + // 512 byte to make sure block alignment tests don't + // fail. + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1099511627997184); + EXPECT_EQ(fptr, nullptr); + EXPECT_EQ(access(testfile.c_str(), F_OK), -1); + EXPECT_EQ(errno, ENOENT); +} + +TEST_F(FiemapWriterTest, CreateUnalignedFile) { + // Try creating a file of size 4097 bytes which is guaranteed + // to be unaligned to all known block sizes. + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize + 1); + ASSERT_NE(fptr, nullptr); + ASSERT_EQ(fptr->size(), gBlockSize * 2); +} + +TEST_F(FiemapWriterTest, CheckFilePath) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize); + ASSERT_NE(fptr, nullptr); + EXPECT_EQ(fptr->size(), gBlockSize); + EXPECT_EQ(fptr->file_path(), testfile); + EXPECT_EQ(access(testfile.c_str(), F_OK), 0); +} + +TEST_F(FiemapWriterTest, CheckFileSize) { + // Create a large-ish file and test that the expected size matches. + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1024 * 1024 * 16); + ASSERT_NE(fptr, nullptr); + + struct stat s; + ASSERT_EQ(stat(testfile.c_str(), &s), 0); + EXPECT_EQ(static_cast(s.st_size), fptr->size()); +} + +TEST_F(FiemapWriterTest, CheckProgress) { + std::vector expected; + size_t invocations = 0; + auto callback = [&](uint64_t done, uint64_t total) -> bool { + if (invocations >= expected.size()) { + return false; + } + EXPECT_EQ(done, expected[invocations]); + EXPECT_EQ(total, gBlockSize); + invocations++; + return true; + }; + + expected.push_back(gBlockSize); + + auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback)); + EXPECT_NE(ptr, nullptr); + EXPECT_EQ(invocations, expected.size()); +} + +TEST_F(FiemapWriterTest, CheckPinning) { + auto ptr = FiemapWriter::Open(testfile, 4096); + ASSERT_NE(ptr, nullptr); + EXPECT_TRUE(FiemapWriter::HasPinnedExtents(testfile)); +} + +TEST_F(FiemapWriterTest, CheckBlockDevicePath) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize); + EXPECT_EQ(fptr->size(), gBlockSize); + EXPECT_EQ(fptr->bdev_path().find("/dev/block/"), size_t(0)); + EXPECT_EQ(fptr->bdev_path().find("/dev/block/dm-"), string::npos); +} + +TEST_F(FiemapWriterTest, CheckFileCreated) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 32768); + ASSERT_NE(fptr, nullptr); + unique_fd fd(open(testfile.c_str(), O_RDONLY)); + EXPECT_GT(fd, -1); +} + +TEST_F(FiemapWriterTest, CheckFileSizeActual) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size); + ASSERT_NE(fptr, nullptr); + + struct stat sb; + ASSERT_EQ(stat(testfile.c_str(), &sb), 0); + EXPECT_GE(sb.st_size, testfile_size); +} + +TEST_F(FiemapWriterTest, CheckFileExtents) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size); + ASSERT_NE(fptr, nullptr); + EXPECT_GT(fptr->extents().size(), 0); +} + +TEST_F(FiemapWriterTest, ExistingFile) { + // Create the file. + { ASSERT_NE(FiemapWriter::Open(testfile, gBlockSize), nullptr); } + // Test that we can still open it. + { + auto ptr = FiemapWriter::Open(testfile, 0, false); + ASSERT_NE(ptr, nullptr); + EXPECT_GT(ptr->extents().size(), 0); + } +} + +TEST_F(FiemapWriterTest, FileDeletedOnError) { + auto callback = [](uint64_t, uint64_t) -> bool { return false; }; + auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback)); + EXPECT_EQ(ptr, nullptr); + EXPECT_EQ(access(testfile.c_str(), F_OK), -1); + EXPECT_EQ(errno, ENOENT); +} + +TEST_F(FiemapWriterTest, MaxBlockSize) { + ASSERT_GT(DetermineMaximumFileSize(testfile), 0); +} + +TEST_F(FiemapWriterTest, FibmapBlockAddressing) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize); + ASSERT_NE(fptr, nullptr); + + switch (fptr->fs_type()) { + case F2FS_SUPER_MAGIC: + case EXT4_SUPER_MAGIC: + // Skip the test for FIEMAP supported filesystems. This is really + // because f2fs/ext4 have caches that seem to defeat reading back + // directly from the block device, and writing directly is too + // dangerous. + std::cout << "Skipping test, filesystem does not use FIBMAP\n"; + return; + } + + bool uses_dm; + std::string bdev_path; + ASSERT_TRUE(FiemapWriter::GetBlockDeviceForFile(testfile, &bdev_path, &uses_dm)); + + if (uses_dm) { + // We could use a device-mapper wrapper here to bypass encryption, but + // really this test is for FIBMAP correctness on VFAT (where encryption + // is never used), so we don't bother. + std::cout << "Skipping test, block device is metadata encrypted\n"; + return; + } + + std::string data(fptr->size(), '\0'); + for (size_t i = 0; i < data.size(); i++) { + data[i] = 'A' + static_cast(data.size() % 26); + } + + { + unique_fd fd(open(testfile.c_str(), O_WRONLY | O_CLOEXEC)); + ASSERT_GE(fd, 0); + ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size())); + ASSERT_EQ(fsync(fd), 0); + } + + ASSERT_FALSE(fptr->extents().empty()); + const auto& first_extent = fptr->extents()[0]; + + unique_fd bdev(open(fptr->bdev_path().c_str(), O_RDONLY | O_CLOEXEC)); + ASSERT_GE(bdev, 0); + + off_t where = first_extent.fe_physical; + ASSERT_EQ(lseek(bdev, where, SEEK_SET), where); + + // Note: this will fail on encrypted folders. + std::string actual(data.size(), '\0'); + ASSERT_GE(first_extent.fe_length, data.size()); + ASSERT_TRUE(android::base::ReadFully(bdev, actual.data(), actual.size())); + EXPECT_EQ(memcmp(actual.data(), data.data(), data.size()), 0); +} + +TEST_F(SplitFiemapTest, Create) { + auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32); + ASSERT_NE(ptr, nullptr); + + auto extents = ptr->extents(); + + // Destroy the fiemap, closing file handles. This should not delete them. + ptr = nullptr; + + std::vector files; + ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files)); + for (const auto& path : files) { + EXPECT_EQ(access(path.c_str(), F_OK), 0); + } + + ASSERT_GE(extents.size(), files.size()); +} + +TEST_F(SplitFiemapTest, Open) { + { + auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32); + ASSERT_NE(ptr, nullptr); + } + + auto ptr = SplitFiemap::Open(testfile); + ASSERT_NE(ptr, nullptr); + + auto extents = ptr->extents(); + ASSERT_GE(extents.size(), 24); +} + +TEST_F(SplitFiemapTest, DeleteOnFail) { + auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 100, 1); + ASSERT_EQ(ptr, nullptr); + + std::string first_file = testfile + ".0001"; + ASSERT_NE(access(first_file.c_str(), F_OK), 0); + ASSERT_EQ(errno, ENOENT); + ASSERT_NE(access(testfile.c_str(), F_OK), 0); + ASSERT_EQ(errno, ENOENT); +} + +static string ReadSplitFiles(const std::string& base_path, size_t num_files) { + std::string result; + for (int i = 0; i < num_files; i++) { + std::string path = base_path + android::base::StringPrintf(".%04d", i); + std::string data; + if (!android::base::ReadFileToString(path, &data)) { + return {}; + } + result += data; + } + return result; +} + +TEST_F(SplitFiemapTest, WriteWholeFile) { + static constexpr size_t kChunkSize = 32768; + static constexpr size_t kSize = kChunkSize * 3; + auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); + ASSERT_NE(ptr, nullptr); + + auto buffer = std::make_unique(kSize / sizeof(int)); + for (size_t i = 0; i < kSize / sizeof(int); i++) { + buffer[i] = i; + } + ASSERT_TRUE(ptr->Write(buffer.get(), kSize)); + + std::string expected(reinterpret_cast(buffer.get()), kSize); + auto actual = ReadSplitFiles(testfile, 3); + ASSERT_EQ(expected.size(), actual.size()); + EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0); +} + +TEST_F(SplitFiemapTest, WriteFileInChunks1) { + static constexpr size_t kChunkSize = 32768; + static constexpr size_t kSize = kChunkSize * 3; + auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); + ASSERT_NE(ptr, nullptr); + + auto buffer = std::make_unique(kSize / sizeof(int)); + for (size_t i = 0; i < kSize / sizeof(int); i++) { + buffer[i] = i; + } + + // Write in chunks of 1000 (so some writes straddle the boundary of two + // files). + size_t bytes_written = 0; + while (bytes_written < kSize) { + size_t to_write = std::min(kSize - bytes_written, (size_t)1000); + char* data = reinterpret_cast(buffer.get()) + bytes_written; + ASSERT_TRUE(ptr->Write(data, to_write)); + bytes_written += to_write; + } + + std::string expected(reinterpret_cast(buffer.get()), kSize); + auto actual = ReadSplitFiles(testfile, 3); + ASSERT_EQ(expected.size(), actual.size()); + EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0); +} + +TEST_F(SplitFiemapTest, WriteFileInChunks2) { + static constexpr size_t kChunkSize = 32768; + static constexpr size_t kSize = kChunkSize * 3; + auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); + ASSERT_NE(ptr, nullptr); + + auto buffer = std::make_unique(kSize / sizeof(int)); + for (size_t i = 0; i < kSize / sizeof(int); i++) { + buffer[i] = i; + } + + // Write in chunks of 32KiB so every write is exactly at the end of the + // current file. + size_t bytes_written = 0; + while (bytes_written < kSize) { + size_t to_write = std::min(kSize - bytes_written, kChunkSize); + char* data = reinterpret_cast(buffer.get()) + bytes_written; + ASSERT_TRUE(ptr->Write(data, to_write)); + bytes_written += to_write; + } + + std::string expected(reinterpret_cast(buffer.get()), kSize); + auto actual = ReadSplitFiles(testfile, 3); + ASSERT_EQ(expected.size(), actual.size()); + EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0); +} + +TEST_F(SplitFiemapTest, WritePastEnd) { + static constexpr size_t kChunkSize = 32768; + static constexpr size_t kSize = kChunkSize * 3; + auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); + ASSERT_NE(ptr, nullptr); + + auto buffer = std::make_unique(kSize / sizeof(int)); + for (size_t i = 0; i < kSize / sizeof(int); i++) { + buffer[i] = i; + } + ASSERT_TRUE(ptr->Write(buffer.get(), kSize)); + ASSERT_FALSE(ptr->Write(buffer.get(), kSize)); +} + +class VerifyBlockWritesExt4 : public ::testing::Test { + // 2GB Filesystem and 4k block size by default + static constexpr uint64_t block_size = 4096; + static constexpr uint64_t fs_size = 2147483648; + + protected: + void SetUp() override { + fs_path = std::string(getenv("TMPDIR")) + "/ext4_2G.img"; + uint64_t count = fs_size / block_size; + std::string dd_cmd = + ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64 + " count=%" PRIu64 " > /dev/null 2>&1", + fs_path.c_str(), block_size, count); + std::string mkfs_cmd = + ::android::base::StringPrintf("/system/bin/mkfs.ext4 -q %s", fs_path.c_str()); + // create mount point + mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt"; + ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0); + // create file for the file system + int ret = system(dd_cmd.c_str()); + ASSERT_EQ(ret, 0); + // Get and attach a loop device to the filesystem we created + LoopDevice loop_dev(fs_path); + ASSERT_TRUE(loop_dev.valid()); + // create file system + ret = system(mkfs_cmd.c_str()); + ASSERT_EQ(ret, 0); + + // mount the file system + ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "ext4", 0, nullptr), 0); + } + + void TearDown() override { + umount(mntpoint.c_str()); + rmdir(mntpoint.c_str()); + unlink(fs_path.c_str()); + } + + std::string mntpoint; + std::string fs_path; +}; + +class VerifyBlockWritesF2fs : public ::testing::Test { + // 2GB Filesystem and 4k block size by default + static constexpr uint64_t block_size = 4096; + static constexpr uint64_t fs_size = 2147483648; + + protected: + void SetUp() override { + fs_path = std::string(getenv("TMPDIR")) + "/f2fs_2G.img"; + uint64_t count = fs_size / block_size; + std::string dd_cmd = + ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64 + " count=%" PRIu64 " > /dev/null 2>&1", + fs_path.c_str(), block_size, count); + std::string mkfs_cmd = + ::android::base::StringPrintf("/system/bin/make_f2fs -q %s", fs_path.c_str()); + // create mount point + mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt"; + ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0); + // create file for the file system + int ret = system(dd_cmd.c_str()); + ASSERT_EQ(ret, 0); + // Get and attach a loop device to the filesystem we created + LoopDevice loop_dev(fs_path); + ASSERT_TRUE(loop_dev.valid()); + // create file system + ret = system(mkfs_cmd.c_str()); + ASSERT_EQ(ret, 0); + + // mount the file system + ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "f2fs", 0, nullptr), 0); + } + + void TearDown() override { + umount(mntpoint.c_str()); + rmdir(mntpoint.c_str()); + unlink(fs_path.c_str()); + } + + std::string mntpoint; + std::string fs_path; +}; + +bool DetermineBlockSize() { + struct statfs s; + if (statfs(gTestDir.c_str(), &s)) { + std::cerr << "Could not call statfs: " << strerror(errno) << "\n"; + return false; + } + if (!s.f_bsize) { + std::cerr << "Invalid block size: " << s.f_bsize << "\n"; + return false; + } + + gBlockSize = s.f_bsize; + return true; +} + +} // namespace fiemap_writer +} // namespace android + +using namespace android::fiemap_writer; + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + if (argc > 1 && argv[1] == "-h"s) { + cerr << "Usage: [test_dir] [file_size]\n"; + cerr << "\n"; + cerr << "Note: test_dir must be a writable, unencrypted directory.\n"; + exit(EXIT_FAILURE); + } + ::android::base::InitLogging(argv, ::android::base::StderrLogger); + + std::string root_dir = "/data/local/unencrypted"; + if (access(root_dir.c_str(), F_OK)) { + root_dir = "/data"; + } + + std::string tempdir = root_dir + "/XXXXXX"s; + if (!mkdtemp(tempdir.data())) { + cerr << "unable to create tempdir on " << root_dir << "\n"; + exit(EXIT_FAILURE); + } + if (!android::base::Realpath(tempdir, &gTestDir)) { + cerr << "unable to find realpath for " << tempdir; + exit(EXIT_FAILURE); + } + + if (argc > 2) { + testfile_size = strtoull(argv[2], NULL, 0); + if (testfile_size == ULLONG_MAX) { + testfile_size = 512 * 1024 * 1024; + } + } + + if (!DetermineBlockSize()) { + exit(EXIT_FAILURE); + } + + auto result = RUN_ALL_TESTS(); + + std::string cmd = "rm -rf " + gTestDir; + system(cmd.c_str()); + + return result; +} diff --git a/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h b/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h new file mode 100644 index 000000000..ee79262a7 --- /dev/null +++ b/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +#include + +namespace android { +namespace fiemap_writer { + +class FiemapWriter; +using FiemapUniquePtr = std::unique_ptr; + +class FiemapWriter final { + public: + // Factory method for FiemapWriter. + // The method returns FiemapUniquePtr that contains all the data necessary to be able to write + // to the given file directly using raw block i/o. The optional progress callback will be + // invoked, if create is true, while the file is being initialized. It receives the bytes + // written and the number of total bytes. If the callback returns false, the operation will + // fail. + // + // Note: when create is true, the file size will be aligned up to the nearest file system + // block. + static FiemapUniquePtr Open(const std::string& file_path, uint64_t file_size, + bool create = true, + std::function progress = {}); + + // Check that a file still has the same extents since it was last opened with FiemapWriter, + // assuming the file was not resized outside of FiemapWriter. Returns false either on error + // or if the file was not pinned. + // + // This will always return true on Ext4. On F2FS, it will return true if either of the + // following cases are true: + // - The file was never pinned. + // - The file is pinned and has not been moved by the GC. + // Thus, this method should only be called for pinned files (such as those returned by + // FiemapWriter::Open). + static bool HasPinnedExtents(const std::string& file_path); + + // Returns the underlying block device of a file. This will look past device-mapper layers + // as long as each layer would not change block mappings (i.e., dm-crypt, dm-bow, and dm- + // default-key tables are okay; dm-linear is not). If a mapping such as dm-linear is found, + // it will be returned in place of any physical block device. + // + // It is the caller's responsibility to check whether the returned block device is acceptable. + // Gsid, for example, will only accept /dev/block/by-name/userdata as the bottom device. + // Callers can check the device name (dm- or loop prefix), inspect sysfs, or compare the major + // number against a boot device. + // + // If device-mapper nodes were encountered, then |uses_dm| will be set to true. + static bool GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path, + bool* uses_dm = nullptr); + + ~FiemapWriter() = default; + + const std::string& file_path() const { return file_path_; }; + uint64_t size() const { return file_size_; }; + const std::string& bdev_path() const { return bdev_path_; }; + uint64_t block_size() const { return block_size_; }; + const std::vector& extents() { return extents_; }; + uint32_t fs_type() const { return fs_type_; } + + // Non-copyable & Non-movable + FiemapWriter(const FiemapWriter&) = delete; + FiemapWriter& operator=(const FiemapWriter&) = delete; + FiemapWriter& operator=(FiemapWriter&&) = delete; + FiemapWriter(FiemapWriter&&) = delete; + + private: + // Name of the file managed by this class. + std::string file_path_; + // Block device on which we have created the file. + std::string bdev_path_; + + // Size in bytes of the file this class is writing + uint64_t file_size_; + + // total size in bytes of the block device + uint64_t bdev_size_; + + // Filesystem type where the file is being created. + // See: for filesystem magic numbers + uint32_t fs_type_; + + // block size as reported by the kernel of the underlying block device; + uint64_t block_size_; + + // This file's fiemap + std::vector extents_; + + FiemapWriter() = default; +}; + +} // namespace fiemap_writer +} // namespace android diff --git a/fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h b/fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h new file mode 100644 index 000000000..7b977e1bd --- /dev/null +++ b/fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include "fiemap_writer.h" + +namespace android { +namespace fiemap_writer { + +// Wrapper around FiemapWriter that is able to split images across files if +// necessary. +class SplitFiemap final { + public: + using ProgressCallback = std::function; + + // Create a new split fiemap file. If |max_piece_size| is 0, the number of + // pieces will be determined automatically by detecting the filesystem. + // Otherwise, the file will be split evenly (with the remainder in the + // final file). + static std::unique_ptr Create(const std::string& file_path, uint64_t file_size, + uint64_t max_piece_size, + ProgressCallback progress = {}); + + // Open an existing split fiemap file. + static std::unique_ptr Open(const std::string& file_path); + + ~SplitFiemap(); + + // Return a list of all files created for a split file. + static bool GetSplitFileList(const std::string& file_path, std::vector* list); + + // Destroy all components of a split file. If the root file does not exist, + // this returns true and does not report an error. + static bool RemoveSplitFiles(const std::string& file_path, std::string* message = nullptr); + + // Return whether all components of a split file still have pinned extents. + bool HasPinnedExtents() const; + + // Helper method for writing data that spans files. Note there is no seek + // method (yet); this starts at 0 and increments the position by |bytes|. + bool Write(const void* data, uint64_t bytes); + + // Flush all writes to all split files. + bool Flush(); + + const std::vector& extents(); + uint32_t block_size() const; + uint64_t size() const { return total_size_; } + const std::string& bdev_path() const; + + // Non-copyable & Non-movable + SplitFiemap(const SplitFiemap&) = delete; + SplitFiemap& operator=(const SplitFiemap&) = delete; + SplitFiemap& operator=(SplitFiemap&&) = delete; + SplitFiemap(SplitFiemap&&) = delete; + + private: + SplitFiemap() = default; + void AddFile(FiemapUniquePtr&& file); + + bool creating_ = false; + std::string list_file_; + std::vector files_; + std::vector extents_; + uint64_t total_size_ = 0; + + // Most recently open file and position for Write(). + size_t cursor_index_ = 0; + uint64_t cursor_file_pos_ = 0; + android::base::unique_fd cursor_fd_; +}; + +} // namespace fiemap_writer +} // namespace android diff --git a/fs_mgr/libfiemap_writer/split_fiemap_writer.cpp b/fs_mgr/libfiemap_writer/split_fiemap_writer.cpp new file mode 100644 index 000000000..16a82d287 --- /dev/null +++ b/fs_mgr/libfiemap_writer/split_fiemap_writer.cpp @@ -0,0 +1,298 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "utility.h" + +namespace android { +namespace fiemap_writer { + +using android::base::unique_fd; + +// We use a four-digit suffix at the end of filenames. +static const size_t kMaxFilePieces = 500; + +std::unique_ptr SplitFiemap::Create(const std::string& file_path, uint64_t file_size, + uint64_t max_piece_size, + ProgressCallback progress) { + if (!file_size) { + LOG(ERROR) << "Cannot create a fiemap for a 0-length file: " << file_path; + return nullptr; + } + + if (!max_piece_size) { + max_piece_size = DetermineMaximumFileSize(file_path); + if (!max_piece_size) { + LOG(ERROR) << "Could not determine maximum file size for " << file_path; + return nullptr; + } + } + + // Call |progress| only when the total percentage would significantly change. + int permille = -1; + uint64_t total_bytes_written = 0; + auto on_progress = [&](uint64_t written, uint64_t) -> bool { + uint64_t actual_written = total_bytes_written + written; + int new_permille = (actual_written * 1000) / file_size; + if (new_permille != permille && actual_written < file_size) { + if (progress && !progress(actual_written, file_size)) { + return false; + } + permille = new_permille; + } + return true; + }; + + std::unique_ptr out(new SplitFiemap()); + out->creating_ = true; + out->list_file_ = file_path; + + // Create the split files. + uint64_t remaining_bytes = file_size; + while (remaining_bytes) { + if (out->files_.size() >= kMaxFilePieces) { + LOG(ERROR) << "Requested size " << file_size << " created too many split files"; + return nullptr; + } + std::string chunk_path = + android::base::StringPrintf("%s.%04d", file_path.c_str(), (int)out->files_.size()); + uint64_t chunk_size = std::min(max_piece_size, remaining_bytes); + auto writer = FiemapWriter::Open(chunk_path, chunk_size, true, on_progress); + if (!writer) { + return nullptr; + } + + // To make sure the alignment doesn't create too much inconsistency, we + // account the *actual* size, not the requested size. + total_bytes_written += writer->size(); + // writer->size() is block size aligned and could be bigger than remaining_bytes + // If remaining_bytes is bigger, set remaining_bytes to 0 to avoid underflow error. + remaining_bytes = remaining_bytes > writer->size() ? (remaining_bytes - writer->size()) : 0; + out->AddFile(std::move(writer)); + } + + // Create the split file list. + unique_fd fd(open(out->list_file_.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0660)); + if (fd < 0) { + PLOG(ERROR) << "Failed to open " << file_path; + return nullptr; + } + + for (const auto& writer : out->files_) { + std::string line = android::base::Basename(writer->file_path()) + "\n"; + if (!android::base::WriteFully(fd, line.data(), line.size())) { + PLOG(ERROR) << "Write failed " << file_path; + return nullptr; + } + } + + // Unset this bit, so we don't unlink on destruction. + out->creating_ = false; + return out; +} + +std::unique_ptr SplitFiemap::Open(const std::string& file_path) { + std::vector files; + if (!GetSplitFileList(file_path, &files)) { + return nullptr; + } + + std::unique_ptr out(new SplitFiemap()); + out->list_file_ = file_path; + + for (const auto& file : files) { + auto writer = FiemapWriter::Open(file, 0, false); + if (!writer) { + // Error was logged in Open(). + return nullptr; + } + out->AddFile(std::move(writer)); + } + return out; +} + +bool SplitFiemap::GetSplitFileList(const std::string& file_path, std::vector* list) { + // This is not the most efficient thing, but it is simple and recovering + // the fiemap/fibmap is much more expensive. + std::string contents; + if (!android::base::ReadFileToString(file_path, &contents, true)) { + PLOG(ERROR) << "Error reading file: " << file_path; + return false; + } + + std::vector names = android::base::Split(contents, "\n"); + std::string dir = android::base::Dirname(file_path); + for (const auto& name : names) { + if (!name.empty()) { + list->emplace_back(dir + "/" + name); + } + } + return true; +} + +bool SplitFiemap::RemoveSplitFiles(const std::string& file_path, std::string* message) { + // Early exit if this does not exist, and do not report an error. + if (access(file_path.c_str(), F_OK) && errno == ENOENT) { + return true; + } + + bool ok = true; + std::vector files; + if (GetSplitFileList(file_path, &files)) { + for (const auto& file : files) { + ok &= android::base::RemoveFileIfExists(file, message); + } + } + ok &= android::base::RemoveFileIfExists(file_path, message); + return ok; +} + +bool SplitFiemap::HasPinnedExtents() const { + for (const auto& file : files_) { + if (!FiemapWriter::HasPinnedExtents(file->file_path())) { + return false; + } + } + return true; +} + +const std::vector& SplitFiemap::extents() { + if (extents_.empty()) { + for (const auto& file : files_) { + const auto& extents = file->extents(); + extents_.insert(extents_.end(), extents.begin(), extents.end()); + } + } + return extents_; +} + +bool SplitFiemap::Write(const void* data, uint64_t bytes) { + // Open the current file. + FiemapWriter* file = files_[cursor_index_].get(); + + const uint8_t* data_ptr = reinterpret_cast(data); + uint64_t bytes_remaining = bytes; + while (bytes_remaining) { + // How many bytes can we write into the current file? + uint64_t file_bytes_left = file->size() - cursor_file_pos_; + if (!file_bytes_left) { + if (cursor_index_ == files_.size() - 1) { + LOG(ERROR) << "write past end of file requested"; + return false; + } + + // No space left in the current file, but we have more files to + // use, so prep the next one. + cursor_fd_ = {}; + cursor_file_pos_ = 0; + file = files_[++cursor_index_].get(); + file_bytes_left = file->size(); + } + + // Open the current file if it's not open. + if (cursor_fd_ < 0) { + cursor_fd_.reset(open(file->file_path().c_str(), O_CLOEXEC | O_WRONLY)); + if (cursor_fd_ < 0) { + PLOG(ERROR) << "open failed: " << file->file_path(); + return false; + } + CHECK(cursor_file_pos_ == 0); + } + + if (!FiemapWriter::HasPinnedExtents(file->file_path())) { + LOG(ERROR) << "file is no longer pinned: " << file->file_path(); + return false; + } + + uint64_t bytes_to_write = std::min(file_bytes_left, bytes_remaining); + if (!android::base::WriteFully(cursor_fd_, data_ptr, bytes_to_write)) { + PLOG(ERROR) << "write failed: " << file->file_path(); + return false; + } + data_ptr += bytes_to_write; + bytes_remaining -= bytes_to_write; + cursor_file_pos_ += bytes_to_write; + } + + // If we've reached the end of the current file, close it for sanity. + if (cursor_file_pos_ == file->size()) { + cursor_fd_ = {}; + } + return true; +} + +bool SplitFiemap::Flush() { + for (const auto& file : files_) { + unique_fd fd(open(file->file_path().c_str(), O_RDONLY | O_CLOEXEC)); + if (fd < 0) { + PLOG(ERROR) << "open failed: " << file->file_path(); + return false; + } + if (fsync(fd)) { + PLOG(ERROR) << "fsync failed: " << file->file_path(); + return false; + } + } + return true; +} + +SplitFiemap::~SplitFiemap() { + if (!creating_) { + return; + } + + // We failed to finish creating, so unlink everything. + unlink(list_file_.c_str()); + for (auto&& file : files_) { + std::string path = file->file_path(); + file = nullptr; + + unlink(path.c_str()); + } +} + +void SplitFiemap::AddFile(FiemapUniquePtr&& file) { + total_size_ += file->size(); + files_.emplace_back(std::move(file)); +} + +uint32_t SplitFiemap::block_size() const { + return files_[0]->block_size(); +} + +const std::string& SplitFiemap::bdev_path() const { + return files_[0]->bdev_path(); +} + +} // namespace fiemap_writer +} // namespace android diff --git a/fs_mgr/libfiemap_writer/testdata/file_32k b/fs_mgr/libfiemap_writer/testdata/file_32k new file mode 100644 index 000000000..12f3be4dd Binary files /dev/null and b/fs_mgr/libfiemap_writer/testdata/file_32k differ diff --git a/fs_mgr/libfiemap_writer/testdata/file_4k b/fs_mgr/libfiemap_writer/testdata/file_4k new file mode 100644 index 000000000..08e7df176 Binary files /dev/null and b/fs_mgr/libfiemap_writer/testdata/file_4k differ diff --git a/fs_mgr/libfiemap_writer/testdata/unaligned_file b/fs_mgr/libfiemap_writer/testdata/unaligned_file new file mode 100644 index 000000000..c107c2637 Binary files /dev/null and b/fs_mgr/libfiemap_writer/testdata/unaligned_file differ diff --git a/fs_mgr/libfiemap_writer/utility.cpp b/fs_mgr/libfiemap_writer/utility.cpp new file mode 100644 index 000000000..192ec1602 --- /dev/null +++ b/fs_mgr/libfiemap_writer/utility.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utility.h" + +#include +#include +#include + +#include +#include + +namespace android { +namespace fiemap_writer { + +uint64_t DetermineMaximumFileSize(const std::string& file_path) { + // Create the smallest file possible (one block). + auto writer = FiemapWriter::Open(file_path, 1); + if (!writer) { + return 0; + } + + uint64_t result = 0; + switch (writer->fs_type()) { + case EXT4_SUPER_MAGIC: + // The minimum is 16GiB, so just report that. If we wanted we could parse the + // superblock and figure out if 64-bit support is enabled. + result = 17179869184ULL; + break; + case F2FS_SUPER_MAGIC: + // Formula is from https://www.kernel.org/doc/Documentation/filesystems/f2fs.txt + // 4KB * (923 + 2 * 1018 + 2 * 1018 * 1018 + 1018 * 1018 * 1018) := 3.94TB. + result = 4329690886144ULL; + break; + case MSDOS_SUPER_MAGIC: + // 4GB-1, which we want aligned to the block size. + result = 4294967295; + result -= (result % writer->block_size()); + break; + default: + LOG(ERROR) << "Unknown file system type: " << writer->fs_type(); + break; + } + + // Close and delete the temporary file. + writer = nullptr; + unlink(file_path.c_str()); + + return result; +} + +} // namespace fiemap_writer +} // namespace android diff --git a/fs_mgr/libfiemap_writer/utility.h b/fs_mgr/libfiemap_writer/utility.h new file mode 100644 index 000000000..2d418dae4 --- /dev/null +++ b/fs_mgr/libfiemap_writer/utility.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace android { +namespace fiemap_writer { + +// Given a file that will be created, determine the maximum size its containing +// filesystem allows. Note this is a theoretical maximum size; free space is +// ignored entirely. +uint64_t DetermineMaximumFileSize(const std::string& file_path); + +} // namespace fiemap_writer +} // namespace android