diff --git a/fs_mgr/libfiemap/Android.bp b/fs_mgr/libfiemap/Android.bp new file mode 100644 index 000000000..8dbbf4c24 --- /dev/null +++ b/fs_mgr/libfiemap/Android.bp @@ -0,0 +1,135 @@ +// +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +cc_library_headers { + name: "libfiemap_headers", + recovery_available: true, + export_include_dirs: ["include"], +} + +cc_defaults { + name: "libfiemap_defaults", + defaults: ["fs_mgr_defaults"], + cflags: [ + "-D_FILE_OFFSET_BITS=64", + "-Wall", + "-Werror", + ], + + srcs: [ + "fiemap_writer.cpp", + "image_manager.cpp", + "metadata.cpp", + "split_fiemap_writer.cpp", + "utility.cpp", + ], + + static_libs: [ + "libdm", + "libext2_uuid", + "libext4_utils", + "liblp", + "libfs_mgr", + ], + + shared_libs: [ + "libbase", + ], + + header_libs: [ + "libfiemap_headers", + "liblog_headers", + ], + + export_shared_lib_headers: [ + "libbase", + ], + + export_header_lib_headers: [ + "libfiemap_headers", + ], +} + +// Open up a binder IImageManager interface. +cc_library_static { + name: "libfiemap_binder", + defaults: ["libfiemap_defaults"], + srcs: [ + "binder.cpp", + ], + whole_static_libs: [ + "gsi_aidl_interface-cpp", + "libgsi", + ], + shared_libs: [ + "libbinder", + ], +} + +// Open up a passthrough IImageManager interface. Use libfiemap_binder whenever +// possible. This should only be used when binder is not available. +cc_library_static { + name: "libfiemap_passthrough", + defaults: ["libfiemap_defaults"], + recovery_available: true, + srcs: [ + "passthrough.cpp", + ], +} + +cc_test { + name: "fiemap_writer_test", + defaults: ["libfiemap_defaults"], + static_libs: [ + "libbase", + "libdm", + "liblog", + ], + + data: [ + "testdata/unaligned_file", + "testdata/file_4k", + "testdata/file_32k", + ], + + srcs: [ + "fiemap_writer_test.cpp", + ], +} + +cc_test { + name: "fiemap_image_test", + defaults: ["libfiemap_defaults"], + static_libs: [ + "libdm", + "libext4_utils", + "libfs_mgr", + "liblp", + ], + shared_libs: [ + "libcrypto", + "libcrypto_utils", + "libcutils", + "liblog", + ], + srcs: [ + "image_test.cpp", + ], +} + +vts_config { + name: "VtsFiemapWriterTest", +} diff --git a/fs_mgr/libfiemap/AndroidTest.xml b/fs_mgr/libfiemap/AndroidTest.xml new file mode 100644 index 000000000..44c80fc50 --- /dev/null +++ b/fs_mgr/libfiemap/AndroidTest.xml @@ -0,0 +1,30 @@ + + + + diff --git a/fs_mgr/libfiemap/README.md b/fs_mgr/libfiemap/README.md new file mode 100644 index 000000000..62d610a5d --- /dev/null +++ b/fs_mgr/libfiemap/README.md @@ -0,0 +1,75 @@ +libfiemap +============= + +`libfiemap` is a library for creating block-devices that are backed by +storage in read-write partitions. It exists primary for gsid. Generally, the +library works by using `libfiemap_writer` to allocate large files within +filesystem, and then tracks their extents. + +There are three main uses for `libfiemap`: + - Creating images that will act as block devices. For example, gsid needs to + create a `system_gsi` image to store Dynamic System Updates. + - Mapping the image as a block device while /data is mounted. This is fairly + tricky and is described in more detail below. + - Mapping the image as a block device during first-stage init. This is simple + because it uses the same logic from dynamic partitions. + +Image creation is done through `SplitFiemap`. Depending on the file system, +a large image may have to be split into multiple files. On Ext4 the limit is +16GiB and on FAT32 it's 4GiB. Images are saved into `/data/gsi//` +where `` is chosen by the process requesting the image. + +At the same time, a file called `/metadata/gsi//lp_metadata` is created. +This is a super partition header that allows first-stage init to create dynamic +partitions from the image files. It also tracks the canonical size of the image, +since the file size may be larger due to alignment. + +Mapping +------- + +It is easy to make block devices out of blocks on `/data` when it is not +mounted, so first-stage init has no issues mapping dynamic partitions from +images. After `/data` is mounted however, there are two problems: + - `/data` is encrypted. + - `/dev/block/by-name/data` may be marked as in-use. + +We break the problem down into three scenarios. + +### FDE and Metadata Encrypted Devices + +When FDE or metadata encryption is used, `/data` is not mounted from +`/dev/block/by-name/data`. Instead, it is mounted from an intermediate +`dm-crypt` or `dm-default-key` device. This means the underlying device is +not marked in use, and we can create new dm-linear devices on top of it. + +On these devices, a block device for an image will consist of a single +device-mapper device with a `dm-linear` table entry for each extent in the +backing file. + +### Unencrypted and FBE-encrypted Devices + +When a device is unencrypted, or is encrypted with FBE but not metadata +encryption, we instead use a loop device with `LOOP_SET_DIRECT_IO` enabled. +Since `/data/gsi` has encryption disabled, this means the raw blocks will be +unencrypted as well. + +### Split Images + +If an image was too large to store a single file on the underlying filesystem, +on an FBE/unencrypted device we will have multiple loop devices. In this case, +we create a device-mapper device as well. For each loop device it will have one +`dm-linear` table entry spanning the length of the device. + +State Tracking +-------------- + +It's important that we know whether or not an image is currently in-use by a +block device. It could be catastrophic to write to a dm-linear device if the +underlying blocks are no longer owned by the original file. Thus, when mapping +an image, we create a property called `gsid.mapped_image.` and set it to +the path of the block device. + +Additionally, we create a `/metadata/gsi//.status` file. Each +line in this file denotes a dependency on either a device-mapper node or a loop +device. When deleting a block device, this file is used to release all +resources. diff --git a/fs_mgr/libfiemap/binder.cpp b/fs_mgr/libfiemap/binder.cpp new file mode 100644 index 000000000..49779f435 --- /dev/null +++ b/fs_mgr/libfiemap/binder.cpp @@ -0,0 +1,224 @@ +// +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#if !defined(__ANDROID_RECOVERY__) +#include +#include +#include +#include +#include +#include +#include + +namespace android { +namespace fiemap { + +using namespace android::gsi; +using namespace std::chrono_literals; + +class ImageManagerBinder final : public IImageManager { + public: + ImageManagerBinder(android::sp&& service, android::sp&& manager); + bool CreateBackingImage(const std::string& name, uint64_t size, int flags) override; + bool DeleteBackingImage(const std::string& name) override; + bool MapImageDevice(const std::string& name, const std::chrono::milliseconds& timeout_ms, + std::string* path) override; + bool UnmapImageDevice(const std::string& name) override; + bool BackingImageExists(const std::string& name) override; + bool IsImageMapped(const std::string& name) override; + bool MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name, + std::string* dev) override; + bool ZeroFillNewImage(const std::string& name, uint64_t bytes) override; + bool RemoveAllImages() override; + + std::vector GetAllBackingImages() override; + + private: + android::sp service_; + android::sp manager_; +}; + +ImageManagerBinder::ImageManagerBinder(android::sp&& service, + android::sp&& manager) + : service_(std::move(service)), manager_(std::move(manager)) {} + +bool ImageManagerBinder::CreateBackingImage(const std::string& name, uint64_t size, int flags) { + auto status = manager_->createBackingImage(name, size, flags); + if (!status.isOk()) { + LOG(ERROR) << __PRETTY_FUNCTION__ + << " binder returned: " << status.exceptionMessage().string(); + return false; + } + return true; +} + +bool ImageManagerBinder::DeleteBackingImage(const std::string& name) { + auto status = manager_->deleteBackingImage(name); + if (!status.isOk()) { + LOG(ERROR) << __PRETTY_FUNCTION__ + << " binder returned: " << status.exceptionMessage().string(); + return false; + } + return true; +} + +bool ImageManagerBinder::MapImageDevice(const std::string& name, + const std::chrono::milliseconds& timeout_ms, + std::string* path) { + int32_t timeout_ms_count = + static_cast(std::clamp( + timeout_ms.count(), INT32_MIN, INT32_MAX)); + MappedImage map; + auto status = manager_->mapImageDevice(name, timeout_ms_count, &map); + if (!status.isOk()) { + LOG(ERROR) << __PRETTY_FUNCTION__ + << " binder returned: " << status.exceptionMessage().string(); + return false; + } + *path = map.path; + return true; +} + +bool ImageManagerBinder::UnmapImageDevice(const std::string& name) { + auto status = manager_->unmapImageDevice(name); + if (!status.isOk()) { + LOG(ERROR) << __PRETTY_FUNCTION__ + << " binder returned: " << status.exceptionMessage().string(); + return false; + } + return true; +} + +bool ImageManagerBinder::BackingImageExists(const std::string& name) { + bool retval; + auto status = manager_->backingImageExists(name, &retval); + if (!status.isOk()) { + LOG(ERROR) << __PRETTY_FUNCTION__ + << " binder returned: " << status.exceptionMessage().string(); + return false; + } + return retval; +} + +bool ImageManagerBinder::IsImageMapped(const std::string& name) { + bool retval; + auto status = manager_->isImageMapped(name, &retval); + if (!status.isOk()) { + LOG(ERROR) << __PRETTY_FUNCTION__ + << " binder returned: " << status.exceptionMessage().string(); + return false; + } + return retval; +} + +bool ImageManagerBinder::MapImageWithDeviceMapper(const IPartitionOpener& opener, + const std::string& name, std::string* dev) { + (void)opener; + (void)name; + (void)dev; + LOG(ERROR) << "MapImageWithDeviceMapper is not available over binder."; + return false; +} + +std::vector ImageManagerBinder::GetAllBackingImages() { + std::vector retval; + auto status = manager_->getAllBackingImages(&retval); + if (!status.isOk()) { + LOG(ERROR) << __PRETTY_FUNCTION__ + << " binder returned: " << status.exceptionMessage().string(); + } + return retval; +} + +bool ImageManagerBinder::ZeroFillNewImage(const std::string& name, uint64_t bytes) { + auto status = manager_->zeroFillNewImage(name, bytes); + if (!status.isOk()) { + LOG(ERROR) << __PRETTY_FUNCTION__ + << " binder returned: " << status.exceptionMessage().string(); + return false; + } + return true; +} + +bool ImageManagerBinder::RemoveAllImages() { + auto status = manager_->removeAllImages(); + if (!status.isOk()) { + LOG(ERROR) << __PRETTY_FUNCTION__ + << " binder returned: " << status.exceptionMessage().string(); + return false; + } + return true; +} + +static android::sp AcquireIGsid(const std::chrono::milliseconds& timeout_ms) { + if (android::base::GetProperty("init.svc.gsid", "") != "running") { + if (!android::base::SetProperty("ctl.start", "gsid") || + !android::base::WaitForProperty("init.svc.gsid", "running", timeout_ms)) { + LOG(ERROR) << "Could not start the gsid service"; + return nullptr; + } + // Sleep for 250ms to give the service time to register. + usleep(250 * 1000); + } + auto sm = android::defaultServiceManager(); + auto name = android::String16(kGsiServiceName); + auto service = sm->checkService(name); + return android::interface_cast(service); +} + +static android::sp GetGsiService(const std::chrono::milliseconds& timeout_ms) { + auto start_time = std::chrono::steady_clock::now(); + + std::chrono::milliseconds elapsed = std::chrono::milliseconds::zero(); + do { + if (auto gsid = AcquireIGsid(timeout_ms - elapsed); gsid != nullptr) { + return gsid; + } + auto now = std::chrono::steady_clock::now(); + elapsed = std::chrono::duration_cast(now - start_time); + } while (elapsed <= timeout_ms); + + LOG(ERROR) << "Timed out trying to acquire IGsid interface"; + return nullptr; +} + +std::unique_ptr IImageManager::Open(const std::string& dir, + const std::chrono::milliseconds& timeout_ms) { + auto gsid = GetGsiService(timeout_ms); + if (!gsid) { + return nullptr; + } + + android::sp service; + auto status = gsid->getClient(&service); + if (!status.isOk() || !service) { + LOG(ERROR) << "Could not acquire IGsiService"; + return nullptr; + } + + android::sp manager; + status = service->openImageService(dir, &manager); + if (!status.isOk() || !manager) { + LOG(ERROR) << "Could not acquire IImageManager: " << status.exceptionMessage().string(); + return nullptr; + } + return std::make_unique(std::move(service), std::move(manager)); +} + +} // namespace fiemap +} // namespace android + +#endif // __ANDROID_RECOVERY__ diff --git a/fs_mgr/libfiemap/fiemap_writer.cpp b/fs_mgr/libfiemap/fiemap_writer.cpp new file mode 100644 index 000000000..961533efa --- /dev/null +++ b/fs_mgr/libfiemap/fiemap_writer.cpp @@ -0,0 +1,782 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include "utility.h" + +namespace android { +namespace fiemap { + +using namespace android::dm; + +// We cap the maximum number of extents as a sanity measure. +static constexpr uint32_t kMaxExtents = 50000; + +// TODO: Fallback to using fibmap if FIEMAP_EXTENT_MERGED is set. +static constexpr const uint32_t kUnsupportedExtentFlags = + FIEMAP_EXTENT_UNKNOWN | FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_DELALLOC | + FIEMAP_EXTENT_NOT_ALIGNED | FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_DATA_TAIL | + FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_SHARED | FIEMAP_EXTENT_MERGED; + +// Large file support must be enabled. +static_assert(sizeof(off_t) == sizeof(uint64_t)); + +static inline void cleanup(const std::string& file_path, bool created) { + if (created) { + unlink(file_path.c_str()); + } +} + +static bool ValidateDmTarget(const DeviceMapper::TargetInfo& target) { + const auto& entry = target.spec; + if (entry.sector_start != 0) { + LOG(INFO) << "Stopping at target with non-zero starting sector"; + return false; + } + + auto target_type = DeviceMapper::GetTargetType(entry); + if (target_type == "bow" || target_type == "default-key" || target_type == "crypt") { + return true; + } + if (target_type == "linear") { + auto pieces = android::base::Split(target.data, " "); + if (pieces[1] != "0") { + LOG(INFO) << "Stopping at complex linear target with non-zero starting sector: " + << pieces[1]; + return false; + } + return true; + } + + LOG(INFO) << "Stopping at complex target type " << target_type; + return false; +} + +static bool DeviceMapperStackPop(const std::string& bdev, std::string* bdev_raw) { + *bdev_raw = bdev; + + if (!::android::base::StartsWith(bdev, "dm-")) { + // We are at the bottom of the device mapper stack. + return true; + } + + // Get the device name. + auto dm_name_file = "/sys/block/" + bdev + "/dm/name"; + std::string dm_name; + if (!android::base::ReadFileToString(dm_name_file, &dm_name)) { + PLOG(ERROR) << "Could not read file: " << dm_name_file; + return false; + } + dm_name = android::base::Trim(dm_name); + + auto& dm = DeviceMapper::Instance(); + std::vector table; + if (!dm.GetTableInfo(dm_name, &table)) { + LOG(ERROR) << "Could not read device-mapper table for " << dm_name << " at " << bdev; + return false; + } + + // The purpose of libfiemap is to provide an extent-based view into + // a file. This is difficult if devices are not layered in a 1:1 manner; + // we would have to translate and break up extents based on the actual + // block mapping. Since this is too complex, we simply stop processing + // the device-mapper stack if we encounter a complex case. + // + // It is up to the caller to decide whether stopping at a virtual block + // device is allowable. In most cases it is not, because we want either + // "userdata" or an external volume. It is useful for tests however. + // Callers can check by comparing the device number to that of userdata, + // or by checking whether is a device-mapper node. + if (table.size() > 1) { + LOG(INFO) << "Stopping at complex table for " << dm_name << " at " << bdev; + return true; + } + if (!ValidateDmTarget(table[0])) { + return true; + } + + auto dm_leaf_dir = "/sys/block/" + bdev + "/slaves"; + auto d = std::unique_ptr(opendir(dm_leaf_dir.c_str()), closedir); + if (d == nullptr) { + PLOG(ERROR) << "Failed to open: " << dm_leaf_dir; + return false; + } + + struct dirent* de; + uint32_t num_leaves = 0; + std::string bdev_next = ""; + while ((de = readdir(d.get())) != nullptr) { + if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) { + continue; + } + + // We set the first name we find here + if (bdev_next.empty()) { + bdev_next = de->d_name; + } + num_leaves++; + } + + // if we have more than one leaves, we return immediately. We can't continue to create the + // file since we don't know how to write it out using fiemap, so it will be readable via the + // underlying block devices later. The reader will also have to construct the same device mapper + // target in order read the file out. + if (num_leaves > 1) { + LOG(ERROR) << "Found " << num_leaves << " leaf block devices under device mapper device " + << bdev; + return false; + } + + // recursively call with the block device we found in order to pop the device mapper stack. + return DeviceMapperStackPop(bdev_next, bdev_raw); +} + +bool FiemapWriter::GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path, + bool* uses_dm) { + struct stat sb; + if (stat(file_path.c_str(), &sb)) { + PLOG(ERROR) << "Failed to get stat for: " << file_path; + return false; + } + + std::string bdev; + if (!BlockDeviceToName(major(sb.st_dev), minor(sb.st_dev), &bdev)) { + LOG(ERROR) << "Failed to get block device name for " << major(sb.st_dev) << ":" + << minor(sb.st_dev); + return false; + } + + std::string bdev_raw; + if (!DeviceMapperStackPop(bdev, &bdev_raw)) { + LOG(ERROR) << "Failed to get the bottom of the device mapper stack for device: " << bdev; + return false; + } + + if (uses_dm) { + *uses_dm = (bdev_raw != bdev); + } + + LOG(DEBUG) << "Popped device (" << bdev_raw << ") from device mapper stack starting with (" + << bdev << ")"; + + *bdev_path = ::android::base::StringPrintf("/dev/block/%s", bdev_raw.c_str()); + + // Make sure we are talking to a block device before calling it a success. + if (stat(bdev_path->c_str(), &sb)) { + PLOG(ERROR) << "Failed to get stat for block device: " << *bdev_path; + return false; + } + + if ((sb.st_mode & S_IFMT) != S_IFBLK) { + PLOG(ERROR) << "File: " << *bdev_path << " is not a block device"; + return false; + } + + return true; +} + +static bool GetBlockDeviceSize(int bdev_fd, const std::string& bdev_path, uint64_t* bdev_size) { + uint64_t size_in_bytes = 0; + if (ioctl(bdev_fd, BLKGETSIZE64, &size_in_bytes)) { + PLOG(ERROR) << "Failed to get total size for: " << bdev_path; + return false; + } + + *bdev_size = size_in_bytes; + + return true; +} + +static uint64_t GetFileSize(const std::string& file_path) { + struct stat sb; + if (stat(file_path.c_str(), &sb)) { + PLOG(ERROR) << "Failed to get size for file: " << file_path; + return 0; + } + + return sb.st_size; +} + +static bool PerformFileChecks(const std::string& file_path, uint64_t file_size, uint64_t* blocksz, + uint32_t* fs_type) { + struct statfs64 sfs; + if (statfs64(file_path.c_str(), &sfs)) { + PLOG(ERROR) << "Failed to read file system status at: " << file_path; + return false; + } + + if (!sfs.f_bsize) { + LOG(ERROR) << "Unsupported block size: " << sfs.f_bsize; + return false; + } + + // Check if the filesystem is of supported types. + // Only ext4, f2fs, and vfat are tested and supported. + switch (sfs.f_type) { + case EXT4_SUPER_MAGIC: + case F2FS_SUPER_MAGIC: + case MSDOS_SUPER_MAGIC: + break; + default: + LOG(ERROR) << "Unsupported file system type: 0x" << std::hex << sfs.f_type; + return false; + } + + uint64_t available_bytes = sfs.f_bsize * sfs.f_bavail; + if (access(file_path.c_str(), F_OK) != 0 && available_bytes <= file_size) { + LOG(ERROR) << "Not enough free space in file system to create file of size : " << file_size; + return false; + } + + *blocksz = sfs.f_bsize; + *fs_type = sfs.f_type; + return true; +} + +static bool FallocateFallback(int file_fd, uint64_t block_size, uint64_t file_size, + const std::string& file_path, + const std::function& on_progress) { + // Even though this is much faster than writing zeroes, it is still slow + // enough that we need to fire the progress callback periodically. To + // easily achieve this, we seek in chunks. We use 1000 chunks since + // normally we only fire the callback on 1/1000th increments. + uint64_t bytes_per_chunk = std::max(file_size / 1000, block_size); + + // Seek just to the end of each chunk and write a single byte, causing + // the filesystem to allocate blocks. + off_t cursor = 0; + off_t end = static_cast(file_size); + while (cursor < end) { + cursor = std::min(static_cast(cursor + bytes_per_chunk), end); + auto rv = TEMP_FAILURE_RETRY(lseek(file_fd, cursor - 1, SEEK_SET)); + if (rv < 0) { + PLOG(ERROR) << "Failed to lseek " << file_path; + return false; + } + if (rv != cursor - 1) { + LOG(ERROR) << "Seek returned wrong offset " << rv << " for file " << file_path; + return false; + } + char buffer[] = {0}; + if (!android::base::WriteFully(file_fd, buffer, 1)) { + PLOG(ERROR) << "Write failed: " << file_path; + return false; + } + if (on_progress && !on_progress(cursor, file_size)) { + return false; + } + } + return true; +} + +// F2FS-specific ioctl +// It requires the below kernel commit merged in v4.16-rc1. +// 1ad71a27124c ("f2fs: add an ioctl to disable GC for specific file") +// In android-4.4, +// 56ee1e817908 ("f2fs: updates on v4.16-rc1") +// In android-4.9, +// 2f17e34672a8 ("f2fs: updates on v4.16-rc1") +// In android-4.14, +// ce767d9a55bc ("f2fs: updates on v4.16-rc1") +#ifndef F2FS_IOC_SET_PIN_FILE +#ifndef F2FS_IOCTL_MAGIC +#define F2FS_IOCTL_MAGIC 0xf5 +#endif +#define F2FS_IOC_GET_PIN_FILE _IOR(F2FS_IOCTL_MAGIC, 14, __u32) +#define F2FS_IOC_SET_PIN_FILE _IOW(F2FS_IOCTL_MAGIC, 13, __u32) +#endif + +static bool IsFilePinned(int file_fd, const std::string& file_path, uint32_t fs_type) { + if (fs_type != F2FS_SUPER_MAGIC) { + // No pinning necessary for ext4 or vfat. The blocks, once allocated, + // are expected to be fixed. + return true; + } + + // f2fs: export FS_NOCOW_FL flag to user + uint32_t flags; + int error = ioctl(file_fd, FS_IOC_GETFLAGS, &flags); + if (error < 0) { + if ((errno == ENOTTY) || (errno == ENOTSUP)) { + PLOG(ERROR) << "Failed to get flags, not supported by kernel: " << file_path; + } else { + PLOG(ERROR) << "Failed to get flags: " << file_path; + } + return false; + } + if (!(flags & FS_NOCOW_FL)) { + return false; + } + + // F2FS_IOC_GET_PIN_FILE returns the number of blocks moved. + uint32_t moved_blocks_nr; + error = ioctl(file_fd, F2FS_IOC_GET_PIN_FILE, &moved_blocks_nr); + if (error < 0) { + if ((errno == ENOTTY) || (errno == ENOTSUP)) { + PLOG(ERROR) << "Failed to get file pin status, not supported by kernel: " << file_path; + } else { + PLOG(ERROR) << "Failed to get file pin status: " << file_path; + } + return false; + } + + if (moved_blocks_nr) { + LOG(WARNING) << moved_blocks_nr << " blocks moved in file " << file_path; + } + return moved_blocks_nr == 0; +} + +static bool PinFile(int file_fd, const std::string& file_path, uint32_t fs_type) { + if (IsFilePinned(file_fd, file_path, fs_type)) { + return true; + } + if (fs_type != F2FS_SUPER_MAGIC) { + // No pinning necessary for ext4/msdos. The blocks, once allocated, are + // expected to be fixed. + return true; + } + + uint32_t pin_status = 1; + int error = ioctl(file_fd, F2FS_IOC_SET_PIN_FILE, &pin_status); + if (error < 0) { + if ((errno == ENOTTY) || (errno == ENOTSUP)) { + PLOG(ERROR) << "Failed to pin file, not supported by kernel: " << file_path; + } else { + PLOG(ERROR) << "Failed to pin file: " << file_path; + } + return false; + } + + return true; +} + +// Reserve space for the file on the file system and write it out to make sure the extents +// don't come back unwritten. Return from this function with the kernel file offset set to 0. +// If the filesystem is f2fs, then we also PIN the file on disk to make sure the blocks +// aren't moved around. +static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blocksz, + uint64_t file_size, unsigned int fs_type, + std::function on_progress) { + switch (fs_type) { + case EXT4_SUPER_MAGIC: + break; + case F2FS_SUPER_MAGIC: { + bool supported; + if (!F2fsPinBeforeAllocate(file_fd, &supported)) { + return false; + } + if (supported && !PinFile(file_fd, file_path, fs_type)) { + return false; + } + break; + } + case MSDOS_SUPER_MAGIC: + // fallocate() is not supported, and not needed, since VFAT does not support holes. + // Instead we can perform a much faster allocation. + return FallocateFallback(file_fd, blocksz, file_size, file_path, on_progress); + default: + LOG(ERROR) << "Missing fallocate() support for file system " << fs_type; + return false; + } + + if (fallocate(file_fd, FALLOC_FL_ZERO_RANGE, 0, file_size)) { + PLOG(ERROR) << "Failed to allocate space for file: " << file_path << " size: " << file_size; + return false; + } + + // write zeroes in 'blocksz' byte increments until we reach file_size to make sure the data + // blocks are actually written to by the file system and thus getting rid of the holes in the + // file. + auto buffer = std::unique_ptr(calloc(1, blocksz), free); + if (buffer == nullptr) { + LOG(ERROR) << "failed to allocate memory for writing file"; + return false; + } + + off64_t offset = lseek64(file_fd, 0, SEEK_SET); + if (offset < 0) { + PLOG(ERROR) << "Failed to seek at the beginning of : " << file_path; + return false; + } + + int permille = -1; + while (offset < file_size) { + if (!::android::base::WriteFully(file_fd, buffer.get(), blocksz)) { + PLOG(ERROR) << "Failed to write" << blocksz << " bytes at offset" << offset + << " in file " << file_path; + return false; + } + + offset += blocksz; + + // Don't invoke the callback every iteration - wait until a significant + // chunk (here, 1/1000th) of the data has been processed. + int new_permille = (static_cast(offset) * 1000) / file_size; + if (new_permille != permille && static_cast(offset) != file_size) { + if (on_progress && !on_progress(offset, file_size)) { + return false; + } + permille = new_permille; + } + } + + if (lseek64(file_fd, 0, SEEK_SET) < 0) { + PLOG(ERROR) << "Failed to reset offset at the beginning of : " << file_path; + return false; + } + + // flush all writes here .. + if (fsync(file_fd)) { + PLOG(ERROR) << "Failed to synchronize written file:" << file_path; + return false; + } + + // Send one last progress notification. + if (on_progress && !on_progress(file_size, file_size)) { + return false; + } + return true; +} + +bool FiemapWriter::HasPinnedExtents(const std::string& file_path) { + android::base::unique_fd fd(open(file_path.c_str(), O_NOFOLLOW | O_CLOEXEC | O_RDONLY)); + if (fd < 0) { + PLOG(ERROR) << "open: " << file_path; + return false; + } + + struct statfs64 sfs; + if (fstatfs64(fd, &sfs)) { + PLOG(ERROR) << "fstatfs64: " << file_path; + return false; + } + return IsFilePinned(fd, file_path, sfs.f_type); +} + +static bool CountFiemapExtents(int file_fd, const std::string& file_path, uint32_t* num_extents) { + struct fiemap fiemap = {}; + fiemap.fm_start = 0; + fiemap.fm_length = UINT64_MAX; + fiemap.fm_flags = FIEMAP_FLAG_SYNC; + fiemap.fm_extent_count = 0; + + if (ioctl(file_fd, FS_IOC_FIEMAP, &fiemap)) { + PLOG(ERROR) << "Failed to get FIEMAP from the kernel for file: " << file_path; + return false; + } + + if (num_extents) { + *num_extents = fiemap.fm_mapped_extents; + } + return true; +} + +static bool IsValidExtent(const fiemap_extent* extent, std::string_view file_path) { + if (extent->fe_flags & kUnsupportedExtentFlags) { + LOG(ERROR) << "Extent at location " << extent->fe_logical << " of file " << file_path + << " has unsupported flags"; + return false; + } + return true; +} + +static bool IsLastExtent(const fiemap_extent* extent) { + if (!(extent->fe_flags & FIEMAP_EXTENT_LAST)) { + LOG(ERROR) << "Extents are being received out-of-order"; + return false; + } + return true; +} + +static bool FiemapToExtents(struct fiemap* fiemap, std::vector* extents, + uint32_t num_extents, std::string_view file_path) { + if (num_extents == 0) return false; + + const struct fiemap_extent* last_extent = &fiemap->fm_extents[num_extents - 1]; + if (!IsLastExtent(last_extent)) { + LOG(ERROR) << "FIEMAP did not return a final extent for file: " << file_path; + return false; + } + + // Iterate through each extent, read and make sure its valid before adding it to the vector + // merging contiguous extents. + fiemap_extent* prev = &fiemap->fm_extents[0]; + if (!IsValidExtent(prev, file_path)) return false; + + for (uint32_t i = 1; i < num_extents; i++) { + fiemap_extent* next = &fiemap->fm_extents[i]; + + // Make sure extents are returned in order + if (next != last_extent && IsLastExtent(next)) return false; + + // Check if extent's flags are valid + if (!IsValidExtent(next, file_path)) return false; + + // Check if the current extent is contiguous with the previous one. + // An extent can be combined with its predecessor only if: + // 1. There is no physical space between the previous and the current + // extent, and + // 2. The physical distance between the previous and current extent + // corresponds to their logical distance (contiguous mapping). + if (prev->fe_physical + prev->fe_length == next->fe_physical && + next->fe_physical - prev->fe_physical == next->fe_logical - prev->fe_logical) { + prev->fe_length += next->fe_length; + } else { + extents->emplace_back(*prev); + prev = next; + } + } + extents->emplace_back(*prev); + + return true; +} + +static bool ReadFiemap(int file_fd, const std::string& file_path, + std::vector* extents) { + uint32_t num_extents; + if (!CountFiemapExtents(file_fd, file_path, &num_extents)) { + return false; + } + if (num_extents == 0) { + LOG(ERROR) << "File " << file_path << " has zero extents"; + return false; + } + if (num_extents > kMaxExtents) { + LOG(ERROR) << "File has " << num_extents << ", maximum is " << kMaxExtents << ": " + << file_path; + return false; + } + + uint64_t fiemap_size = + sizeof(struct fiemap_extent) + num_extents * sizeof(struct fiemap_extent); + auto buffer = std::unique_ptr(calloc(1, fiemap_size), free); + if (buffer == nullptr) { + LOG(ERROR) << "Failed to allocate memory for fiemap"; + return false; + } + + struct fiemap* fiemap = reinterpret_cast(buffer.get()); + fiemap->fm_start = 0; + fiemap->fm_length = UINT64_MAX; + // make sure file is synced to disk before we read the fiemap + fiemap->fm_flags = FIEMAP_FLAG_SYNC; + fiemap->fm_extent_count = num_extents; + + if (ioctl(file_fd, FS_IOC_FIEMAP, fiemap)) { + PLOG(ERROR) << "Failed to get FIEMAP from the kernel for file: " << file_path; + return false; + } + if (fiemap->fm_mapped_extents != num_extents) { + LOG(ERROR) << "FIEMAP returned unexpected extent count (" << num_extents + << " expected, got " << fiemap->fm_mapped_extents << ") for file: " << file_path; + return false; + } + + return FiemapToExtents(fiemap, extents, num_extents, file_path); +} + +static bool ReadFibmap(int file_fd, const std::string& file_path, + std::vector* extents) { + struct stat s; + if (fstat(file_fd, &s)) { + PLOG(ERROR) << "Failed to stat " << file_path; + return false; + } + + unsigned int blksize; + if (ioctl(file_fd, FIGETBSZ, &blksize) < 0) { + PLOG(ERROR) << "Failed to get FIGETBSZ for " << file_path; + return false; + } + if (!blksize) { + LOG(ERROR) << "Invalid filesystem block size: " << blksize; + return false; + } + + uint64_t num_blocks = (s.st_size + blksize - 1) / blksize; + if (num_blocks > std::numeric_limits::max()) { + LOG(ERROR) << "Too many blocks for FIBMAP (" << num_blocks << ")"; + return false; + } + + for (uint32_t last_block, block_number = 0; block_number < num_blocks; block_number++) { + uint32_t block = block_number; + if (ioctl(file_fd, FIBMAP, &block)) { + PLOG(ERROR) << "Failed to get FIBMAP for file " << file_path; + return false; + } + if (!block) { + LOG(ERROR) << "Logical block " << block_number << " is a hole, which is not supported"; + return false; + } + + if (!extents->empty() && block == last_block + 1) { + extents->back().fe_length += blksize; + } else { + extents->push_back(fiemap_extent{.fe_logical = block_number, + .fe_physical = static_cast(block) * blksize, + .fe_length = static_cast(blksize), + .fe_flags = 0}); + if (extents->size() > kMaxExtents) { + LOG(ERROR) << "File has more than " << kMaxExtents << "extents: " << file_path; + return false; + } + } + last_block = block; + } + return true; +} + +FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_size, bool create, + std::function progress) { + // if 'create' is false, open an existing file and do not truncate. + int open_flags = O_RDWR | O_CLOEXEC; + if (create) { + if (access(file_path.c_str(), F_OK) == 0) { + LOG(WARNING) << "File " << file_path << " already exists, truncating"; + } + open_flags |= O_CREAT | O_TRUNC; + } + ::android::base::unique_fd file_fd( + TEMP_FAILURE_RETRY(open(file_path.c_str(), open_flags, S_IRUSR | S_IWUSR))); + if (file_fd < 0) { + PLOG(ERROR) << "Failed to create file at: " << file_path; + return nullptr; + } + + std::string abs_path; + if (!::android::base::Realpath(file_path, &abs_path)) { + PLOG(ERROR) << "Invalid file path: " << file_path; + cleanup(file_path, create); + return nullptr; + } + + std::string bdev_path; + if (!GetBlockDeviceForFile(abs_path, &bdev_path)) { + LOG(ERROR) << "Failed to get block dev path for file: " << file_path; + cleanup(abs_path, create); + return nullptr; + } + + ::android::base::unique_fd bdev_fd( + TEMP_FAILURE_RETRY(open(bdev_path.c_str(), O_RDONLY | O_CLOEXEC))); + if (bdev_fd < 0) { + PLOG(ERROR) << "Failed to open block device: " << bdev_path; + cleanup(file_path, create); + return nullptr; + } + + uint64_t bdevsz; + if (!GetBlockDeviceSize(bdev_fd, bdev_path, &bdevsz)) { + LOG(ERROR) << "Failed to get block device size for : " << bdev_path; + cleanup(file_path, create); + return nullptr; + } + + if (!create) { + file_size = GetFileSize(abs_path); + if (file_size == 0) { + LOG(ERROR) << "Invalid file size of zero bytes for file: " << abs_path; + return nullptr; + } + } + + uint64_t blocksz; + uint32_t fs_type; + if (!PerformFileChecks(abs_path, file_size, &blocksz, &fs_type)) { + LOG(ERROR) << "Failed to validate file or file system for file:" << abs_path; + cleanup(abs_path, create); + return nullptr; + } + + // Align up to the nearest block size. + if (file_size % blocksz) { + file_size += blocksz - (file_size % blocksz); + } + + if (create) { + if (!AllocateFile(file_fd, abs_path, blocksz, file_size, fs_type, std::move(progress))) { + LOG(ERROR) << "Failed to allocate file: " << abs_path << " of size: " << file_size + << " bytes"; + cleanup(abs_path, create); + return nullptr; + } + } + + // f2fs may move the file blocks around. + if (!PinFile(file_fd, abs_path, fs_type)) { + cleanup(abs_path, create); + LOG(ERROR) << "Failed to pin the file in storage"; + return nullptr; + } + + // now allocate the FiemapWriter and start setting it up + FiemapUniquePtr fmap(new FiemapWriter()); + switch (fs_type) { + case EXT4_SUPER_MAGIC: + case F2FS_SUPER_MAGIC: + if (!ReadFiemap(file_fd, abs_path, &fmap->extents_)) { + LOG(ERROR) << "Failed to read fiemap of file: " << abs_path; + cleanup(abs_path, create); + return nullptr; + } + break; + case MSDOS_SUPER_MAGIC: + if (!ReadFibmap(file_fd, abs_path, &fmap->extents_)) { + LOG(ERROR) << "Failed to read fibmap of file: " << abs_path; + cleanup(abs_path, create); + return nullptr; + } + break; + } + + fmap->file_path_ = abs_path; + fmap->bdev_path_ = bdev_path; + fmap->file_size_ = file_size; + fmap->bdev_size_ = bdevsz; + fmap->fs_type_ = fs_type; + fmap->block_size_ = blocksz; + + LOG(VERBOSE) << "Successfully created FiemapWriter for file " << abs_path << " on block device " + << bdev_path; + return fmap; +} + +} // namespace fiemap +} // namespace android diff --git a/fs_mgr/libfiemap/fiemap_writer_test.cpp b/fs_mgr/libfiemap/fiemap_writer_test.cpp new file mode 100644 index 000000000..4ac7161c5 --- /dev/null +++ b/fs_mgr/libfiemap/fiemap_writer_test.cpp @@ -0,0 +1,541 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utility.h" + +namespace android { +namespace fiemap { + +using namespace std; +using namespace std::string_literals; +using namespace android::fiemap; +using unique_fd = android::base::unique_fd; +using LoopDevice = android::dm::LoopDevice; + +std::string gTestDir; +uint64_t testfile_size = 536870912; // default of 512MiB +size_t gBlockSize = 0; + +class FiemapWriterTest : public ::testing::Test { + protected: + void SetUp() override { + const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info(); + testfile = gTestDir + "/"s + tinfo->name(); + } + + void TearDown() override { unlink(testfile.c_str()); } + + // name of the file we use for testing + std::string testfile; +}; + +class SplitFiemapTest : public ::testing::Test { + protected: + void SetUp() override { + const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info(); + testfile = gTestDir + "/"s + tinfo->name(); + } + + void TearDown() override { + std::string message; + if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) { + cerr << "Could not remove all split files: " << message; + } + } + + // name of the file we use for testing + std::string testfile; +}; + +TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) { + // Try creating a file of size ~100TB but aligned to + // 512 byte to make sure block alignment tests don't + // fail. + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1099511627997184); + EXPECT_EQ(fptr, nullptr); + EXPECT_EQ(access(testfile.c_str(), F_OK), -1); + EXPECT_EQ(errno, ENOENT); +} + +TEST_F(FiemapWriterTest, CreateUnalignedFile) { + // Try creating a file of size 4097 bytes which is guaranteed + // to be unaligned to all known block sizes. + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize + 1); + ASSERT_NE(fptr, nullptr); + ASSERT_EQ(fptr->size(), gBlockSize * 2); +} + +TEST_F(FiemapWriterTest, CheckFilePath) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize); + ASSERT_NE(fptr, nullptr); + EXPECT_EQ(fptr->size(), gBlockSize); + EXPECT_EQ(fptr->file_path(), testfile); + EXPECT_EQ(access(testfile.c_str(), F_OK), 0); +} + +TEST_F(FiemapWriterTest, CheckFileSize) { + // Create a large-ish file and test that the expected size matches. + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1024 * 1024 * 16); + ASSERT_NE(fptr, nullptr); + + struct stat s; + ASSERT_EQ(stat(testfile.c_str(), &s), 0); + EXPECT_EQ(static_cast(s.st_size), fptr->size()); +} + +TEST_F(FiemapWriterTest, CheckProgress) { + std::vector expected; + size_t invocations = 0; + auto callback = [&](uint64_t done, uint64_t total) -> bool { + if (invocations >= expected.size()) { + return false; + } + EXPECT_EQ(done, expected[invocations]); + EXPECT_EQ(total, gBlockSize); + invocations++; + return true; + }; + + expected.push_back(gBlockSize); + + auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback)); + EXPECT_NE(ptr, nullptr); + EXPECT_EQ(invocations, expected.size()); +} + +TEST_F(FiemapWriterTest, CheckPinning) { + auto ptr = FiemapWriter::Open(testfile, 4096); + ASSERT_NE(ptr, nullptr); + EXPECT_TRUE(FiemapWriter::HasPinnedExtents(testfile)); +} + +TEST_F(FiemapWriterTest, CheckBlockDevicePath) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize); + EXPECT_EQ(fptr->size(), gBlockSize); + EXPECT_EQ(fptr->bdev_path().find("/dev/block/"), size_t(0)); + EXPECT_EQ(fptr->bdev_path().find("/dev/block/dm-"), string::npos); +} + +TEST_F(FiemapWriterTest, CheckFileCreated) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 32768); + ASSERT_NE(fptr, nullptr); + unique_fd fd(open(testfile.c_str(), O_RDONLY)); + EXPECT_GT(fd, -1); +} + +TEST_F(FiemapWriterTest, CheckFileSizeActual) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size); + ASSERT_NE(fptr, nullptr); + + struct stat sb; + ASSERT_EQ(stat(testfile.c_str(), &sb), 0); + EXPECT_GE(sb.st_size, testfile_size); +} + +TEST_F(FiemapWriterTest, CheckFileExtents) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size); + ASSERT_NE(fptr, nullptr); + EXPECT_GT(fptr->extents().size(), 0); +} + +TEST_F(FiemapWriterTest, ExistingFile) { + // Create the file. + { ASSERT_NE(FiemapWriter::Open(testfile, gBlockSize), nullptr); } + // Test that we can still open it. + { + auto ptr = FiemapWriter::Open(testfile, 0, false); + ASSERT_NE(ptr, nullptr); + EXPECT_GT(ptr->extents().size(), 0); + } +} + +TEST_F(FiemapWriterTest, FileDeletedOnError) { + auto callback = [](uint64_t, uint64_t) -> bool { return false; }; + auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback)); + EXPECT_EQ(ptr, nullptr); + EXPECT_EQ(access(testfile.c_str(), F_OK), -1); + EXPECT_EQ(errno, ENOENT); +} + +TEST_F(FiemapWriterTest, MaxBlockSize) { + ASSERT_GT(DetermineMaximumFileSize(testfile), 0); +} + +TEST_F(FiemapWriterTest, FibmapBlockAddressing) { + FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize); + ASSERT_NE(fptr, nullptr); + + switch (fptr->fs_type()) { + case F2FS_SUPER_MAGIC: + case EXT4_SUPER_MAGIC: + // Skip the test for FIEMAP supported filesystems. This is really + // because f2fs/ext4 have caches that seem to defeat reading back + // directly from the block device, and writing directly is too + // dangerous. + std::cout << "Skipping test, filesystem does not use FIBMAP\n"; + return; + } + + bool uses_dm; + std::string bdev_path; + ASSERT_TRUE(FiemapWriter::GetBlockDeviceForFile(testfile, &bdev_path, &uses_dm)); + + if (uses_dm) { + // We could use a device-mapper wrapper here to bypass encryption, but + // really this test is for FIBMAP correctness on VFAT (where encryption + // is never used), so we don't bother. + std::cout << "Skipping test, block device is metadata encrypted\n"; + return; + } + + std::string data(fptr->size(), '\0'); + for (size_t i = 0; i < data.size(); i++) { + data[i] = 'A' + static_cast(data.size() % 26); + } + + { + unique_fd fd(open(testfile.c_str(), O_WRONLY | O_CLOEXEC)); + ASSERT_GE(fd, 0); + ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size())); + ASSERT_EQ(fsync(fd), 0); + } + + ASSERT_FALSE(fptr->extents().empty()); + const auto& first_extent = fptr->extents()[0]; + + unique_fd bdev(open(fptr->bdev_path().c_str(), O_RDONLY | O_CLOEXEC)); + ASSERT_GE(bdev, 0); + + off_t where = first_extent.fe_physical; + ASSERT_EQ(lseek(bdev, where, SEEK_SET), where); + + // Note: this will fail on encrypted folders. + std::string actual(data.size(), '\0'); + ASSERT_GE(first_extent.fe_length, data.size()); + ASSERT_TRUE(android::base::ReadFully(bdev, actual.data(), actual.size())); + EXPECT_EQ(memcmp(actual.data(), data.data(), data.size()), 0); +} + +TEST_F(SplitFiemapTest, Create) { + auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32); + ASSERT_NE(ptr, nullptr); + + auto extents = ptr->extents(); + + // Destroy the fiemap, closing file handles. This should not delete them. + ptr = nullptr; + + std::vector files; + ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files)); + for (const auto& path : files) { + EXPECT_EQ(access(path.c_str(), F_OK), 0); + } + + ASSERT_GE(extents.size(), files.size()); +} + +TEST_F(SplitFiemapTest, Open) { + { + auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32); + ASSERT_NE(ptr, nullptr); + } + + auto ptr = SplitFiemap::Open(testfile); + ASSERT_NE(ptr, nullptr); + + auto extents = ptr->extents(); + ASSERT_GE(extents.size(), 24); +} + +TEST_F(SplitFiemapTest, DeleteOnFail) { + auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 100, 1); + ASSERT_EQ(ptr, nullptr); + + std::string first_file = testfile + ".0001"; + ASSERT_NE(access(first_file.c_str(), F_OK), 0); + ASSERT_EQ(errno, ENOENT); + ASSERT_NE(access(testfile.c_str(), F_OK), 0); + ASSERT_EQ(errno, ENOENT); +} + +static string ReadSplitFiles(const std::string& base_path, size_t num_files) { + std::string result; + for (int i = 0; i < num_files; i++) { + std::string path = base_path + android::base::StringPrintf(".%04d", i); + std::string data; + if (!android::base::ReadFileToString(path, &data)) { + return {}; + } + result += data; + } + return result; +} + +TEST_F(SplitFiemapTest, WriteWholeFile) { + static constexpr size_t kChunkSize = 32768; + static constexpr size_t kSize = kChunkSize * 3; + auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); + ASSERT_NE(ptr, nullptr); + + auto buffer = std::make_unique(kSize / sizeof(int)); + for (size_t i = 0; i < kSize / sizeof(int); i++) { + buffer[i] = i; + } + ASSERT_TRUE(ptr->Write(buffer.get(), kSize)); + + std::string expected(reinterpret_cast(buffer.get()), kSize); + auto actual = ReadSplitFiles(testfile, 3); + ASSERT_EQ(expected.size(), actual.size()); + EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0); +} + +TEST_F(SplitFiemapTest, WriteFileInChunks1) { + static constexpr size_t kChunkSize = 32768; + static constexpr size_t kSize = kChunkSize * 3; + auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); + ASSERT_NE(ptr, nullptr); + + auto buffer = std::make_unique(kSize / sizeof(int)); + for (size_t i = 0; i < kSize / sizeof(int); i++) { + buffer[i] = i; + } + + // Write in chunks of 1000 (so some writes straddle the boundary of two + // files). + size_t bytes_written = 0; + while (bytes_written < kSize) { + size_t to_write = std::min(kSize - bytes_written, (size_t)1000); + char* data = reinterpret_cast(buffer.get()) + bytes_written; + ASSERT_TRUE(ptr->Write(data, to_write)); + bytes_written += to_write; + } + + std::string expected(reinterpret_cast(buffer.get()), kSize); + auto actual = ReadSplitFiles(testfile, 3); + ASSERT_EQ(expected.size(), actual.size()); + EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0); +} + +TEST_F(SplitFiemapTest, WriteFileInChunks2) { + static constexpr size_t kChunkSize = 32768; + static constexpr size_t kSize = kChunkSize * 3; + auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); + ASSERT_NE(ptr, nullptr); + + auto buffer = std::make_unique(kSize / sizeof(int)); + for (size_t i = 0; i < kSize / sizeof(int); i++) { + buffer[i] = i; + } + + // Write in chunks of 32KiB so every write is exactly at the end of the + // current file. + size_t bytes_written = 0; + while (bytes_written < kSize) { + size_t to_write = std::min(kSize - bytes_written, kChunkSize); + char* data = reinterpret_cast(buffer.get()) + bytes_written; + ASSERT_TRUE(ptr->Write(data, to_write)); + bytes_written += to_write; + } + + std::string expected(reinterpret_cast(buffer.get()), kSize); + auto actual = ReadSplitFiles(testfile, 3); + ASSERT_EQ(expected.size(), actual.size()); + EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0); +} + +TEST_F(SplitFiemapTest, WritePastEnd) { + static constexpr size_t kChunkSize = 32768; + static constexpr size_t kSize = kChunkSize * 3; + auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize); + ASSERT_NE(ptr, nullptr); + + auto buffer = std::make_unique(kSize / sizeof(int)); + for (size_t i = 0; i < kSize / sizeof(int); i++) { + buffer[i] = i; + } + ASSERT_TRUE(ptr->Write(buffer.get(), kSize)); + ASSERT_FALSE(ptr->Write(buffer.get(), kSize)); +} + +class VerifyBlockWritesExt4 : public ::testing::Test { + // 2GB Filesystem and 4k block size by default + static constexpr uint64_t block_size = 4096; + static constexpr uint64_t fs_size = 2147483648; + + protected: + void SetUp() override { + fs_path = std::string(getenv("TMPDIR")) + "/ext4_2G.img"; + uint64_t count = fs_size / block_size; + std::string dd_cmd = + ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64 + " count=%" PRIu64 " > /dev/null 2>&1", + fs_path.c_str(), block_size, count); + std::string mkfs_cmd = + ::android::base::StringPrintf("/system/bin/mkfs.ext4 -q %s", fs_path.c_str()); + // create mount point + mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt"; + ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0); + // create file for the file system + int ret = system(dd_cmd.c_str()); + ASSERT_EQ(ret, 0); + // Get and attach a loop device to the filesystem we created + LoopDevice loop_dev(fs_path, 10s); + ASSERT_TRUE(loop_dev.valid()); + // create file system + ret = system(mkfs_cmd.c_str()); + ASSERT_EQ(ret, 0); + + // mount the file system + ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "ext4", 0, nullptr), 0); + } + + void TearDown() override { + umount(mntpoint.c_str()); + rmdir(mntpoint.c_str()); + unlink(fs_path.c_str()); + } + + std::string mntpoint; + std::string fs_path; +}; + +class VerifyBlockWritesF2fs : public ::testing::Test { + // 2GB Filesystem and 4k block size by default + static constexpr uint64_t block_size = 4096; + static constexpr uint64_t fs_size = 2147483648; + + protected: + void SetUp() override { + fs_path = std::string(getenv("TMPDIR")) + "/f2fs_2G.img"; + uint64_t count = fs_size / block_size; + std::string dd_cmd = + ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64 + " count=%" PRIu64 " > /dev/null 2>&1", + fs_path.c_str(), block_size, count); + std::string mkfs_cmd = + ::android::base::StringPrintf("/system/bin/make_f2fs -q %s", fs_path.c_str()); + // create mount point + mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt"; + ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0); + // create file for the file system + int ret = system(dd_cmd.c_str()); + ASSERT_EQ(ret, 0); + // Get and attach a loop device to the filesystem we created + LoopDevice loop_dev(fs_path, 10s); + ASSERT_TRUE(loop_dev.valid()); + // create file system + ret = system(mkfs_cmd.c_str()); + ASSERT_EQ(ret, 0); + + // mount the file system + ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "f2fs", 0, nullptr), 0); + } + + void TearDown() override { + umount(mntpoint.c_str()); + rmdir(mntpoint.c_str()); + unlink(fs_path.c_str()); + } + + std::string mntpoint; + std::string fs_path; +}; + +bool DetermineBlockSize() { + struct statfs s; + if (statfs(gTestDir.c_str(), &s)) { + std::cerr << "Could not call statfs: " << strerror(errno) << "\n"; + return false; + } + if (!s.f_bsize) { + std::cerr << "Invalid block size: " << s.f_bsize << "\n"; + return false; + } + + gBlockSize = s.f_bsize; + return true; +} + +} // namespace fiemap +} // namespace android + +using namespace android::fiemap; + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + if (argc > 1 && argv[1] == "-h"s) { + cerr << "Usage: [test_dir] [file_size]\n"; + cerr << "\n"; + cerr << "Note: test_dir must be a writable, unencrypted directory.\n"; + exit(EXIT_FAILURE); + } + ::android::base::InitLogging(argv, ::android::base::StderrLogger); + + std::string root_dir = "/data/local/unencrypted"; + if (access(root_dir.c_str(), F_OK)) { + root_dir = "/data"; + } + + std::string tempdir = root_dir + "/XXXXXX"s; + if (!mkdtemp(tempdir.data())) { + cerr << "unable to create tempdir on " << root_dir << "\n"; + exit(EXIT_FAILURE); + } + if (!android::base::Realpath(tempdir, &gTestDir)) { + cerr << "unable to find realpath for " << tempdir; + exit(EXIT_FAILURE); + } + + if (argc > 2) { + testfile_size = strtoull(argv[2], NULL, 0); + if (testfile_size == ULLONG_MAX) { + testfile_size = 512 * 1024 * 1024; + } + } + + if (!DetermineBlockSize()) { + exit(EXIT_FAILURE); + } + + auto result = RUN_ALL_TESTS(); + + std::string cmd = "rm -rf " + gTestDir; + system(cmd.c_str()); + + return result; +} diff --git a/fs_mgr/libfiemap/image_manager.cpp b/fs_mgr/libfiemap/image_manager.cpp new file mode 100644 index 000000000..fe2018d4d --- /dev/null +++ b/fs_mgr/libfiemap/image_manager.cpp @@ -0,0 +1,674 @@ +// +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "metadata.h" +#include "utility.h" + +namespace android { +namespace fiemap { + +using namespace std::literals; +using android::base::unique_fd; +using android::dm::DeviceMapper; +using android::dm::DmDeviceState; +using android::dm::DmTable; +using android::dm::DmTargetLinear; +using android::dm::LoopControl; +using android::fs_mgr::CreateLogicalPartition; +using android::fs_mgr::CreateLogicalPartitionParams; +using android::fs_mgr::DestroyLogicalPartition; +using android::fs_mgr::GetPartitionName; + +static constexpr char kTestImageMetadataDir[] = "/metadata/gsi/test"; + +std::unique_ptr ImageManager::Open(const std::string& dir_prefix) { + auto metadata_dir = "/metadata/gsi/" + dir_prefix; + auto data_dir = "/data/gsi/" + dir_prefix; + return Open(metadata_dir, data_dir); +} + +std::unique_ptr ImageManager::Open(const std::string& metadata_dir, + const std::string& data_dir) { + return std::unique_ptr(new ImageManager(metadata_dir, data_dir)); +} + +ImageManager::ImageManager(const std::string& metadata_dir, const std::string& data_dir) + : metadata_dir_(metadata_dir), data_dir_(data_dir) { + partition_opener_ = std::make_unique(); +} + +std::string ImageManager::GetImageHeaderPath(const std::string& name) { + return JoinPaths(data_dir_, name) + ".img"; +} + +// The status file has one entry per line, with each entry formatted as one of: +// dm: +// loop: +// +// This simplifies the process of tearing down a mapping, since we can simply +// unmap each entry in the order it appears. +std::string ImageManager::GetStatusFilePath(const std::string& image_name) { + return JoinPaths(metadata_dir_, image_name) + ".status"; +} + +static std::string GetStatusPropertyName(const std::string& image_name) { + // Note: we don't prefix |image_name|, because CreateLogicalPartition won't + // prefix the name either. There are no plans to change this at the moment, + // consumers of the image API must take care to use globally-unique image + // names. + return "gsid.mapped_image." + image_name; +} + +void ImageManager::set_partition_opener(std::unique_ptr&& opener) { + partition_opener_ = std::move(opener); +} + +bool ImageManager::IsImageMapped(const std::string& image_name) { + auto prop_name = GetStatusPropertyName(image_name); + if (android::base::GetProperty(prop_name, "").empty()) { + // If mapped in first-stage init, the dm-device will exist but not the + // property. + auto& dm = DeviceMapper::Instance(); + return dm.GetState(image_name) != DmDeviceState::INVALID; + } + return true; +} + +std::vector ImageManager::GetAllBackingImages() { + std::vector images; + if (!MetadataExists(metadata_dir_)) { + return images; + } + auto metadata = OpenMetadata(metadata_dir_); + if (metadata) { + for (auto&& partition : metadata->partitions) { + images.push_back(partition.name); + } + } + return images; +} + +bool ImageManager::PartitionExists(const std::string& name) { + if (!MetadataExists(metadata_dir_)) { + return false; + } + auto metadata = OpenMetadata(metadata_dir_); + if (!metadata) { + return false; + } + return !!FindPartition(*metadata.get(), name); +} + +bool ImageManager::BackingImageExists(const std::string& name) { + auto header_file = GetImageHeaderPath(name); + return access(header_file.c_str(), F_OK) == 0; +} + +bool ImageManager::CreateBackingImage(const std::string& name, uint64_t size, int flags) { + return CreateBackingImage(name, size, flags, nullptr); +} + +static bool IsUnreliablePinningAllowed(const std::string& path) { + return android::base::StartsWith(path, "/data/gsi/dsu/") || + android::base::StartsWith(path, "/data/gsi/test/") || + android::base::StartsWith(path, "/data/gsi/ota/test/"); +} + +bool ImageManager::CreateBackingImage(const std::string& name, uint64_t size, int flags, + std::function&& on_progress) { + auto data_path = GetImageHeaderPath(name); + auto fw = SplitFiemap::Create(data_path, size, 0, on_progress); + if (!fw) { + return false; + } + + bool reliable_pinning; + if (!FilesystemHasReliablePinning(data_path, &reliable_pinning)) { + return false; + } + if (!reliable_pinning && !IsUnreliablePinningAllowed(data_path)) { + // For historical reasons, we allow unreliable pinning for certain use + // cases (DSUs, testing) because the ultimate use case is either + // developer-oriented or ephemeral (the intent is to boot immediately + // into DSUs). For everything else - such as snapshots/OTAs or adb + // remount, we have a higher bar, and require the filesystem to support + // proper pinning. + LOG(ERROR) << "File system does not have reliable block pinning"; + SplitFiemap::RemoveSplitFiles(data_path); + return false; + } + + // Except for testing, we do not allow persisting metadata that references + // device-mapper devices. It just doesn't make sense, because the device + // numbering may change on reboot. We allow it for testing since the images + // are not meant to survive reboot. Outside of tests, this can only happen + // if device-mapper is stacked in some complex way not supported by + // FiemapWriter. + auto device_path = GetDevicePathForFile(fw.get()); + if (android::base::StartsWith(device_path, "/dev/block/dm-") && + !android::base::StartsWith(metadata_dir_, kTestImageMetadataDir)) { + LOG(ERROR) << "Cannot persist images against device-mapper device: " << device_path; + + fw = {}; + SplitFiemap::RemoveSplitFiles(data_path); + return false; + } + + bool readonly = !!(flags & CREATE_IMAGE_READONLY); + if (!UpdateMetadata(metadata_dir_, name, fw.get(), size, readonly)) { + return false; + } + + if (flags & CREATE_IMAGE_ZERO_FILL) { + if (!ZeroFillNewImage(name, 0)) { + DeleteBackingImage(name); + return false; + } + } + return true; +} + +bool ImageManager::ZeroFillNewImage(const std::string& name, uint64_t bytes) { + auto data_path = GetImageHeaderPath(name); + + // See the comment in MapImageDevice() about how this works. + std::string block_device; + bool can_use_devicemapper; + if (!FiemapWriter::GetBlockDeviceForFile(data_path, &block_device, &can_use_devicemapper)) { + LOG(ERROR) << "Could not determine block device for " << data_path; + return false; + } + + if (!can_use_devicemapper) { + // We've backed with loop devices, and since we store files in an + // unencrypted folder, the initial zeroes we wrote will suffice. + return true; + } + + // data is dm-crypt, or FBE + dm-default-key. This means the zeroes written + // by libfiemap were encrypted, so we need to map the image in and correct + // this. + auto device = MappedDevice::Open(this, 10s, name); + if (!device) { + return false; + } + + static constexpr size_t kChunkSize = 4096; + std::string zeroes(kChunkSize, '\0'); + + uint64_t remaining; + if (bytes) { + remaining = bytes; + } else { + remaining = get_block_device_size(device->fd()); + if (!remaining) { + PLOG(ERROR) << "Could not get block device size for " << device->path(); + return false; + } + } + while (remaining) { + uint64_t to_write = std::min(static_cast(zeroes.size()), remaining); + if (!android::base::WriteFully(device->fd(), zeroes.data(), + static_cast(to_write))) { + PLOG(ERROR) << "write failed: " << device->path(); + return false; + } + remaining -= to_write; + } + return true; +} + +bool ImageManager::DeleteBackingImage(const std::string& name) { + // For dm-linear devices sitting on top of /data, we cannot risk deleting + // the file. The underlying blocks could be reallocated by the filesystem. + if (IsImageMapped(name)) { + LOG(ERROR) << "Backing image " << name << " is currently mapped to a block device"; + return false; + } + + std::string message; + auto header_file = GetImageHeaderPath(name); + if (!SplitFiemap::RemoveSplitFiles(header_file, &message)) { + // This is fatal, because we don't want to leave these files dangling. + LOG(ERROR) << "Error removing image " << name << ": " << message; + return false; + } + + auto status_file = GetStatusFilePath(name); + if (!android::base::RemoveFileIfExists(status_file)) { + LOG(ERROR) << "Error removing " << status_file << ": " << message; + } + return RemoveImageMetadata(metadata_dir_, name); +} + +// Create a block device for an image file, using its extents in its +// lp_metadata. +bool ImageManager::MapWithDmLinear(const IPartitionOpener& opener, const std::string& name, + const std::chrono::milliseconds& timeout_ms, std::string* path) { + // :TODO: refresh extents in metadata file until f2fs is fixed. + auto metadata = OpenMetadata(metadata_dir_); + if (!metadata) { + return false; + } + + auto super = android::fs_mgr::GetMetadataSuperBlockDevice(*metadata.get()); + auto block_device = android::fs_mgr::GetBlockDevicePartitionName(*super); + + CreateLogicalPartitionParams params = { + .block_device = block_device, + .metadata = metadata.get(), + .partition_name = name, + .force_writable = true, + .timeout_ms = timeout_ms, + .partition_opener = &opener, + }; + if (!CreateLogicalPartition(params, path)) { + LOG(ERROR) << "Error creating device-mapper node for image " << name; + return false; + } + + auto status_string = "dm:" + name; + auto status_file = GetStatusFilePath(name); + if (!android::base::WriteStringToFile(status_string, status_file)) { + PLOG(ERROR) << "Could not write status file: " << status_file; + DestroyLogicalPartition(name); + return false; + } + return true; +} + +// Helper to create a loop device for a file. +static bool CreateLoopDevice(LoopControl& control, const std::string& file, + const std::chrono::milliseconds& timeout_ms, std::string* path) { + static constexpr int kOpenFlags = O_RDWR | O_NOFOLLOW | O_CLOEXEC; + android::base::unique_fd file_fd(open(file.c_str(), kOpenFlags)); + if (file_fd < 0) { + PLOG(ERROR) << "Could not open file: " << file; + return false; + } + if (!control.Attach(file_fd, timeout_ms, path)) { + LOG(ERROR) << "Could not create loop device for: " << file; + return false; + } + LOG(INFO) << "Created loop device " << *path << " for file " << file; + return true; +} + +class AutoDetachLoopDevices final { + public: + AutoDetachLoopDevices(LoopControl& control, const std::vector& devices) + : control_(control), devices_(devices), commit_(false) {} + + ~AutoDetachLoopDevices() { + if (commit_) return; + for (const auto& device : devices_) { + control_.Detach(device); + } + } + + void Commit() { commit_ = true; } + + private: + LoopControl& control_; + const std::vector& devices_; + bool commit_; +}; + +// If an image is stored across multiple files, this takes a list of loop +// devices and joins them together using device-mapper. +bool ImageManager::MapWithLoopDeviceList(const std::vector& device_list, + const std::string& name, + const std::chrono::milliseconds& timeout_ms, + std::string* path) { + auto metadata = OpenMetadata(metadata_dir_); + if (!metadata) { + return false; + } + auto partition = FindPartition(*metadata.get(), name); + if (!partition) { + LOG(ERROR) << "Could not find image in metadata: " << name; + return false; + } + + // Since extent lengths are in sector units, the size should be a multiple + // of the sector size. + uint64_t partition_size = GetPartitionSize(*metadata.get(), *partition); + if (partition_size % LP_SECTOR_SIZE != 0) { + LOG(ERROR) << "Partition size not sector aligned: " << name << ", " << partition_size + << " bytes"; + return false; + } + + DmTable table; + + uint64_t start_sector = 0; + uint64_t sectors_needed = partition_size / LP_SECTOR_SIZE; + for (const auto& block_device : device_list) { + // The final block device must be == partition_size, otherwise we + // can't find the AVB footer on verified partitions. + static constexpr int kOpenFlags = O_RDWR | O_NOFOLLOW | O_CLOEXEC; + unique_fd fd(open(block_device.c_str(), kOpenFlags)); + if (fd < 0) { + PLOG(ERROR) << "Open failed: " << block_device; + return false; + } + + uint64_t file_size = get_block_device_size(fd); + uint64_t file_sectors = file_size / LP_SECTOR_SIZE; + uint64_t segment_size = std::min(file_sectors, sectors_needed); + + table.Emplace(start_sector, segment_size, block_device, 0); + + start_sector += segment_size; + sectors_needed -= segment_size; + if (sectors_needed == 0) { + break; + } + } + + auto& dm = DeviceMapper::Instance(); + if (!dm.CreateDevice(name, table, path, timeout_ms)) { + LOG(ERROR) << "Could not create device-mapper device over loop set"; + return false; + } + + // Build the status file. + std::vector lines; + lines.emplace_back("dm:" + name); + for (const auto& block_device : device_list) { + lines.emplace_back("loop:" + block_device); + } + auto status_message = android::base::Join(lines, "\n"); + auto status_file = GetStatusFilePath(name); + if (!android::base::WriteStringToFile(status_message, status_file)) { + PLOG(ERROR) << "Write failed: " << status_file; + dm.DeleteDevice(name); + return false; + } + return true; +} + +static bool OptimizeLoopDevices(const std::vector& device_list) { + for (const auto& device : device_list) { + unique_fd fd(open(device.c_str(), O_RDWR | O_CLOEXEC | O_NOFOLLOW)); + if (fd < 0) { + PLOG(ERROR) << "Open failed: " << device; + return false; + } + if (!LoopControl::EnableDirectIo(fd)) { + return false; + } + } + return true; +} + +// Helper to use one or more loop devices around image files. +bool ImageManager::MapWithLoopDevice(const std::string& name, + const std::chrono::milliseconds& timeout_ms, + std::string* path) { + auto image_header = GetImageHeaderPath(name); + + std::vector file_list; + if (!SplitFiemap::GetSplitFileList(image_header, &file_list)) { + LOG(ERROR) << "Could not get image file list"; + return false; + } + + // Map each image file as a loopback device. + LoopControl control; + std::vector loop_devices; + AutoDetachLoopDevices auto_detach(control, loop_devices); + + auto start_time = std::chrono::steady_clock::now(); + for (const auto& file : file_list) { + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(now - start_time); + + std::string loop_device; + if (!CreateLoopDevice(control, file, timeout_ms - elapsed, &loop_device)) { + break; + } + loop_devices.emplace_back(loop_device); + } + if (loop_devices.size() != file_list.size()) { + // The number of devices will mismatch if CreateLoopDevice() failed. + return false; + } + + // If OptimizeLoopDevices fails, we'd use double the memory. + if (!OptimizeLoopDevices(loop_devices)) { + return false; + } + + // If there's only one loop device (by far the most common case, splits + // will normally only happen on sdcards with FAT32), then just return that + // as the block device. Otherwise, we need to use dm-linear to stitch + // together all the loop devices we just created. + if (loop_devices.size() > 1) { + if (!MapWithLoopDeviceList(loop_devices, name, timeout_ms, path)) { + return false; + } + } + + auto status_message = "loop:" + loop_devices.back(); + auto status_file = GetStatusFilePath(name); + if (!android::base::WriteStringToFile(status_message, status_file)) { + PLOG(ERROR) << "Write failed: " << status_file; + return false; + } + + auto_detach.Commit(); + + *path = loop_devices.back(); + return true; +} + +bool ImageManager::MapImageDevice(const std::string& name, + const std::chrono::milliseconds& timeout_ms, std::string* path) { + if (IsImageMapped(name)) { + LOG(ERROR) << "Backing image " << name << " is already mapped"; + return false; + } + + auto image_header = GetImageHeaderPath(name); + + // If there is a device-mapper node wrapping the block device, then we're + // able to create another node around it; the dm layer does not carry the + // exclusion lock down the stack when a mount occurs. + // + // If there is no intermediate device-mapper node, then partitions cannot be + // opened writable due to sepolicy and exclusivity of having a mounted + // filesystem. This should only happen on devices with no encryption, or + // devices with FBE and no metadata encryption. For these cases it suffices + // to perform normal file writes to /data/gsi (which is unencrypted). + std::string block_device; + bool can_use_devicemapper; + if (!FiemapWriter::GetBlockDeviceForFile(image_header, &block_device, &can_use_devicemapper)) { + LOG(ERROR) << "Could not determine block device for " << image_header; + return false; + } + + if (can_use_devicemapper) { + if (!MapWithDmLinear(*partition_opener_.get(), name, timeout_ms, path)) { + return false; + } + } else if (!MapWithLoopDevice(name, timeout_ms, path)) { + return false; + } + + // Set a property so we remember this is mapped. + auto prop_name = GetStatusPropertyName(name); + if (!android::base::SetProperty(prop_name, *path)) { + UnmapImageDevice(name, true); + return false; + } + return true; +} + +bool ImageManager::MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name, + std::string* dev) { + std::string ignore_path; + if (!MapWithDmLinear(opener, name, {}, &ignore_path)) { + return false; + } + + auto& dm = DeviceMapper::Instance(); + if (!dm.GetDeviceString(name, dev)) { + return false; + } + return true; +} + +bool ImageManager::UnmapImageDevice(const std::string& name) { + return UnmapImageDevice(name, false); +} + +bool ImageManager::UnmapImageDevice(const std::string& name, bool force) { + if (!force && !IsImageMapped(name)) { + LOG(ERROR) << "Backing image " << name << " is not mapped"; + return false; + } + auto& dm = DeviceMapper::Instance(); + LoopControl loop; + + std::string status; + auto status_file = GetStatusFilePath(name); + if (!android::base::ReadFileToString(status_file, &status)) { + PLOG(ERROR) << "Read failed: " << status_file; + return false; + } + + auto lines = android::base::Split(status, "\n"); + for (const auto& line : lines) { + auto pieces = android::base::Split(line, ":"); + if (pieces.size() != 2) { + LOG(ERROR) << "Unknown status line"; + continue; + } + if (pieces[0] == "dm") { + // Failure to remove a dm node is fatal, since we can't safely + // remove the file or loop devices. + const auto& name = pieces[1]; + if (!dm.DeleteDeviceIfExists(name)) { + return false; + } + } else if (pieces[0] == "loop") { + // Failure to remove a loop device is not fatal, since we can still + // remove the backing file if we want. + loop.Detach(pieces[1]); + } else { + LOG(ERROR) << "Unknown status: " << pieces[0]; + } + } + + std::string message; + if (!android::base::RemoveFileIfExists(status_file, &message)) { + LOG(ERROR) << "Could not remove " << status_file << ": " << message; + } + + auto status_prop = GetStatusPropertyName(name); + android::base::SetProperty(status_prop, ""); + return true; +} + +bool ImageManager::RemoveAllImages() { + if (!MetadataExists(metadata_dir_)) { + return true; + } + auto metadata = OpenMetadata(metadata_dir_); + if (!metadata) { + return RemoveAllMetadata(metadata_dir_); + } + + bool ok = true; + for (const auto& partition : metadata->partitions) { + auto partition_name = GetPartitionName(partition); + ok &= DeleteBackingImage(partition_name); + } + return ok && RemoveAllMetadata(metadata_dir_); +} + +bool ImageManager::Validate() { + auto metadata = OpenMetadata(metadata_dir_); + if (!metadata) { + return false; + } + + for (const auto& partition : metadata->partitions) { + auto name = GetPartitionName(partition); + auto image_path = GetImageHeaderPath(name); + auto fiemap = SplitFiemap::Open(image_path); + if (!fiemap || !fiemap->HasPinnedExtents()) { + LOG(ERROR) << "Image is missing or was moved: " << image_path; + return false; + } + } + return true; +} + +std::unique_ptr MappedDevice::Open(IImageManager* manager, + const std::chrono::milliseconds& timeout_ms, + const std::string& name) { + std::string path; + if (!manager->MapImageDevice(name, timeout_ms, &path)) { + return nullptr; + } + + auto device = std::unique_ptr(new MappedDevice(manager, name, path)); + if (device->fd() < 0) { + return nullptr; + } + return device; +} + +MappedDevice::MappedDevice(IImageManager* manager, const std::string& name, const std::string& path) + : manager_(manager), name_(name), path_(path) { + // The device is already mapped; try and open it. + fd_.reset(open(path.c_str(), O_RDWR | O_CLOEXEC)); +} + +MappedDevice::~MappedDevice() { + fd_ = {}; + manager_->UnmapImageDevice(name_); +} + +bool IImageManager::UnmapImageIfExists(const std::string& name) { + // No lock is needed even though this seems to be vulnerable to TOCTOU. If process A + // calls MapImageDevice() while process B calls UnmapImageIfExists(), and MapImageDevice() + // happens after process B checks IsImageMapped(), it would be as if MapImageDevice() is called + // after process B finishes calling UnmapImageIfExists(), resulting the image to be mapped, + // which is a reasonable sequence. + if (!IsImageMapped(name)) { + return true; + } + return UnmapImageDevice(name); +} + +} // namespace fiemap +} // namespace android diff --git a/fs_mgr/libfiemap/image_test.cpp b/fs_mgr/libfiemap/image_test.cpp new file mode 100644 index 000000000..f05825c88 --- /dev/null +++ b/fs_mgr/libfiemap/image_test.cpp @@ -0,0 +1,251 @@ +// +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace android::dm; +using namespace std::literals; +using android::base::unique_fd; +using android::fiemap::ImageManager; +using android::fs_mgr::BlockDeviceInfo; +using android::fs_mgr::PartitionOpener; +using android::fs_mgr::WaitForFile; + +static std::string gDataPath; +static std::string gDataMountPath; +static constexpr char kMetadataPath[] = "/metadata/gsi/test"; + +static constexpr uint64_t kTestImageSize = 1024 * 1024; + +class TestPartitionOpener final : public PartitionOpener { + public: + android::base::unique_fd Open(const std::string& partition_name, int flags) const override { + return PartitionOpener::Open(GetPathForBlockDeviceName(partition_name), flags); + } + bool GetInfo(const std::string& partition_name, BlockDeviceInfo* info) const override { + return PartitionOpener::GetInfo(GetPathForBlockDeviceName(partition_name), info); + } + std::string GetDeviceString(const std::string& partition_name) const override { + return PartitionOpener::GetDeviceString(GetPathForBlockDeviceName(partition_name)); + } + + private: + static std::string GetPathForBlockDeviceName(const std::string& name) { + if (android::base::StartsWith(name, "loop") || android::base::StartsWith(name, "dm-")) { + return "/dev/block/"s + name; + } + return name; + } +}; + +// This fixture is for tests against the device's native configuration. +class NativeTest : public ::testing::Test { + protected: + void SetUp() override { + manager_ = ImageManager::Open(kMetadataPath, gDataPath); + ASSERT_NE(manager_, nullptr); + + manager_->set_partition_opener(std::make_unique()); + + const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info(); + base_name_ = tinfo->name(); + } + + void TearDown() override { + manager_->UnmapImageDevice(base_name_); + manager_->DeleteBackingImage(base_name_); + } + + std::string PropertyName() { return "gsid.mapped_image." + base_name_; } + + std::unique_ptr manager_; + std::string base_name_; +}; + +TEST_F(NativeTest, CreateAndMap) { + ASSERT_TRUE(manager_->CreateBackingImage(base_name_, kTestImageSize, false, nullptr)); + + std::string path; + ASSERT_TRUE(manager_->MapImageDevice(base_name_, 5s, &path)); + ASSERT_TRUE(manager_->IsImageMapped(base_name_)); + ASSERT_EQ(android::base::GetProperty(PropertyName(), ""), path); + + { + unique_fd fd(open(path.c_str(), O_RDWR | O_NOFOLLOW | O_CLOEXEC)); + ASSERT_GE(fd, 0); + ASSERT_EQ(get_block_device_size(fd), kTestImageSize); + } + + ASSERT_TRUE(manager_->UnmapImageDevice(base_name_)); + ASSERT_FALSE(manager_->IsImageMapped(base_name_)); + ASSERT_EQ(android::base::GetProperty(PropertyName(), ""), ""); +} + +// This fixture is for tests against a simulated device environment. Rather +// than use /data, we create an image and then layer a new filesystem within +// it. Each test then decides how to mount and create layered images. This +// allows us to test FBE vs FDE configurations. +class ImageTest : public ::testing::Test { + public: + ImageTest() : dm_(DeviceMapper::Instance()) {} + + void SetUp() override { + manager_ = ImageManager::Open(kMetadataPath, gDataPath); + ASSERT_NE(manager_, nullptr); + + manager_->set_partition_opener(std::make_unique()); + + submanager_ = ImageManager::Open(kMetadataPath + "/mnt"s, gDataPath + "/mnt"s); + ASSERT_NE(submanager_, nullptr); + + submanager_->set_partition_opener(std::make_unique()); + + // Ensure that metadata is cleared in between runs. + submanager_->RemoveAllImages(); + manager_->RemoveAllImages(); + + const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info(); + base_name_ = tinfo->name(); + test_image_name_ = base_name_ + "-base"; + wrapper_device_name_ = base_name_ + "-wrapper"; + + ASSERT_TRUE(manager_->CreateBackingImage(base_name_, kTestImageSize * 16, false, nullptr)); + ASSERT_TRUE(manager_->MapImageDevice(base_name_, 5s, &base_device_)); + } + + void TearDown() override { + submanager_->UnmapImageDevice(test_image_name_); + umount(gDataMountPath.c_str()); + dm_.DeleteDeviceIfExists(wrapper_device_name_); + manager_->UnmapImageDevice(base_name_); + manager_->DeleteBackingImage(base_name_); + } + + protected: + bool DoFormat(const std::string& device) { + // clang-format off + std::vector mkfs_args = { + "/system/bin/mke2fs", + "-F", + "-b 4096", + "-t ext4", + "-m 0", + "-O has_journal", + device, + ">/dev/null", + "2>/dev/null", + " manager_; + std::unique_ptr submanager_; + + DeviceMapper& dm_; + std::string base_name_; + std::string base_device_; + std::string test_image_name_; + std::string wrapper_device_name_; +}; + +TEST_F(ImageTest, DirectMount) { + ASSERT_TRUE(DoFormat(base_device_)); + ASSERT_EQ(mount(base_device_.c_str(), gDataMountPath.c_str(), "ext4", 0, nullptr), 0); + ASSERT_TRUE(submanager_->CreateBackingImage(test_image_name_, kTestImageSize, false, nullptr)); + + std::string path; + ASSERT_TRUE(submanager_->MapImageDevice(test_image_name_, 5s, &path)); + ASSERT_TRUE(android::base::StartsWith(path, "/dev/block/loop")); +} + +TEST_F(ImageTest, IndirectMount) { + // Create a simple wrapper around the base device that we'll mount from + // instead. This will simulate the code paths for dm-crypt/default-key/bow + // and force us to use device-mapper rather than loop devices. + uint64_t device_size = 0; + { + unique_fd fd(open(base_device_.c_str(), O_RDWR | O_CLOEXEC)); + ASSERT_GE(fd, 0); + device_size = get_block_device_size(fd); + ASSERT_EQ(device_size, kTestImageSize * 16); + } + uint64_t num_sectors = device_size / 512; + + auto& dm = DeviceMapper::Instance(); + + DmTable table; + table.Emplace(0, num_sectors, base_device_, 0); + ASSERT_TRUE(dm.CreateDevice(wrapper_device_name_, table)); + + // Format and mount. + std::string wrapper_device; + ASSERT_TRUE(dm.GetDmDevicePathByName(wrapper_device_name_, &wrapper_device)); + ASSERT_TRUE(WaitForFile(wrapper_device, 5s)); + ASSERT_TRUE(DoFormat(wrapper_device)); + ASSERT_EQ(mount(wrapper_device.c_str(), gDataMountPath.c_str(), "ext4", 0, nullptr), 0); + + ASSERT_TRUE(submanager_->CreateBackingImage(test_image_name_, kTestImageSize, false, nullptr)); + + std::string path; + ASSERT_TRUE(submanager_->MapImageDevice(test_image_name_, 5s, &path)); + ASSERT_TRUE(android::base::StartsWith(path, "/dev/block/dm-")); +} + +bool Mkdir(const std::string& path) { + if (mkdir(path.c_str(), 0700) && errno != EEXIST) { + std::cerr << "Could not mkdir " << path << ": " << strerror(errno) << std::endl; + return false; + } + return true; +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + + if (argc >= 2) { + gDataPath = argv[1]; + } else { + gDataPath = "/data/gsi/test"; + } + gDataMountPath = gDataPath + "/mnt"s; + + if (!Mkdir(gDataPath) || !Mkdir(kMetadataPath) || !Mkdir(gDataMountPath) || + !Mkdir(kMetadataPath + "/mnt"s)) { + return 1; + } + return RUN_ALL_TESTS(); +} diff --git a/fs_mgr/libfiemap/include/libfiemap/fiemap_writer.h b/fs_mgr/libfiemap/include/libfiemap/fiemap_writer.h new file mode 100644 index 000000000..c6922658d --- /dev/null +++ b/fs_mgr/libfiemap/include/libfiemap/fiemap_writer.h @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +#include + +namespace android { +namespace fiemap { + +class FiemapWriter; +using FiemapUniquePtr = std::unique_ptr; + +class FiemapWriter final { + public: + // Factory method for FiemapWriter. + // The method returns FiemapUniquePtr that contains all the data necessary to be able to write + // to the given file directly using raw block i/o. The optional progress callback will be + // invoked, if create is true, while the file is being initialized. It receives the bytes + // written and the number of total bytes. If the callback returns false, the operation will + // fail. + // + // Note: when create is true, the file size will be aligned up to the nearest file system + // block. + static FiemapUniquePtr Open(const std::string& file_path, uint64_t file_size, + bool create = true, + std::function progress = {}); + + // Check that a file still has the same extents since it was last opened with FiemapWriter, + // assuming the file was not resized outside of FiemapWriter. Returns false either on error + // or if the file was not pinned. + // + // This will always return true on Ext4. On F2FS, it will return true if either of the + // following cases are true: + // - The file was never pinned. + // - The file is pinned and has not been moved by the GC. + // Thus, this method should only be called for pinned files (such as those returned by + // FiemapWriter::Open). + static bool HasPinnedExtents(const std::string& file_path); + + // Returns the underlying block device of a file. This will look past device-mapper layers + // as long as each layer would not change block mappings (i.e., dm-crypt, dm-bow, and dm- + // default-key tables are okay; dm-linear is not). If a mapping such as dm-linear is found, + // it will be returned in place of any physical block device. + // + // It is the caller's responsibility to check whether the returned block device is acceptable. + // Gsid, for example, will only accept /dev/block/by-name/userdata as the bottom device. + // Callers can check the device name (dm- or loop prefix), inspect sysfs, or compare the major + // number against a boot device. + // + // If device-mapper nodes were encountered, then |uses_dm| will be set to true. + static bool GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path, + bool* uses_dm = nullptr); + + ~FiemapWriter() = default; + + const std::string& file_path() const { return file_path_; }; + uint64_t size() const { return file_size_; }; + const std::string& bdev_path() const { return bdev_path_; }; + uint64_t block_size() const { return block_size_; }; + const std::vector& extents() { return extents_; }; + uint32_t fs_type() const { return fs_type_; } + + // Non-copyable & Non-movable + FiemapWriter(const FiemapWriter&) = delete; + FiemapWriter& operator=(const FiemapWriter&) = delete; + FiemapWriter& operator=(FiemapWriter&&) = delete; + FiemapWriter(FiemapWriter&&) = delete; + + private: + // Name of the file managed by this class. + std::string file_path_; + // Block device on which we have created the file. + std::string bdev_path_; + + // Size in bytes of the file this class is writing + uint64_t file_size_; + + // total size in bytes of the block device + uint64_t bdev_size_; + + // Filesystem type where the file is being created. + // See: for filesystem magic numbers + uint32_t fs_type_; + + // block size as reported by the kernel of the underlying block device; + uint64_t block_size_; + + // This file's fiemap + std::vector extents_; + + FiemapWriter() = default; +}; + +} // namespace fiemap +} // namespace android diff --git a/fs_mgr/libfiemap/include/libfiemap/image_manager.h b/fs_mgr/libfiemap/include/libfiemap/image_manager.h new file mode 100644 index 000000000..5ff462817 --- /dev/null +++ b/fs_mgr/libfiemap/include/libfiemap/image_manager.h @@ -0,0 +1,185 @@ +// +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include + +#include +#include +#include +#include + +#include +#include + +namespace android { +namespace fiemap { + +class IImageManager { + public: + using IPartitionOpener = android::fs_mgr::IPartitionOpener; + + virtual ~IImageManager() {} + + // When linking to libfiemap_binder, the Open() call will use binder. + // Otherwise, the Open() call will use the ImageManager implementation + // below. + static std::unique_ptr Open(const std::string& dir_prefix, + const std::chrono::milliseconds& timeout_ms); + + // Flags for CreateBackingImage(). + static constexpr int CREATE_IMAGE_DEFAULT = 0x0; + static constexpr int CREATE_IMAGE_READONLY = 0x1; + static constexpr int CREATE_IMAGE_ZERO_FILL = 0x2; + + // Create an image that can be mapped as a block-device. If |force_zero_fill| + // is true, the image will be zero-filled. Otherwise, the initial content + // of the image is undefined. If zero-fill is requested, and the operation + // cannot be completed, the image will be deleted and this function will + // return false. + virtual bool CreateBackingImage(const std::string& name, uint64_t size, int flags) = 0; + + // Delete an image created with CreateBackingImage. Its entry will be + // removed from the associated lp_metadata file. + virtual bool DeleteBackingImage(const std::string& name) = 0; + + // Create a block device for an image previously created with + // CreateBackingImage. This will wait for at most |timeout_ms| milliseconds + // for |path| to be available, and will return false if not available in + // the requested time. If |timeout_ms| is zero, this is NOT guaranteed to + // return true. A timeout of 10s is recommended. + // + // Note that snapshots created with a readonly flag are always mapped + // writable. The flag is persisted in the lp_metadata file however, so if + // fs_mgr::CreateLogicalPartition(s) is used, the flag will be respected. + virtual bool MapImageDevice(const std::string& name, + const std::chrono::milliseconds& timeout_ms, std::string* path) = 0; + + // Unmap a block device previously mapped with mapBackingImage. + virtual bool UnmapImageDevice(const std::string& name) = 0; + + // Returns true whether the named backing image exists. + virtual bool BackingImageExists(const std::string& name) = 0; + + // Returns true if the specified image is mapped to a device. + virtual bool IsImageMapped(const std::string& name) = 0; + + // Map an image using device-mapper. This is not available over binder, and + // is intended only for first-stage init. The returned device is a major:minor + // device string. + virtual bool MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name, + std::string* dev) = 0; + + // Get all backing image names. + virtual std::vector GetAllBackingImages() = 0; + + // Writes |bytes| zeros to |name| file. If |bytes| is 0, then the + // whole file if filled with zeros. + virtual bool ZeroFillNewImage(const std::string& name, uint64_t bytes) = 0; + + // Find and remove all images and metadata for this manager. + virtual bool RemoveAllImages() = 0; + + virtual bool UnmapImageIfExists(const std::string& name); +}; + +class ImageManager final : public IImageManager { + public: + // Return an ImageManager for the given metadata and data directories. Both + // directories must already exist. + static std::unique_ptr Open(const std::string& metadata_dir, + const std::string& data_dir); + + // Helper function that derives the metadata and data dirs given a single + // prefix. + static std::unique_ptr Open(const std::string& dir_prefix); + + // Methods that must be implemented from IImageManager. + bool CreateBackingImage(const std::string& name, uint64_t size, int flags) override; + bool DeleteBackingImage(const std::string& name) override; + bool MapImageDevice(const std::string& name, const std::chrono::milliseconds& timeout_ms, + std::string* path) override; + bool UnmapImageDevice(const std::string& name) override; + bool BackingImageExists(const std::string& name) override; + bool IsImageMapped(const std::string& name) override; + bool MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name, + std::string* dev) override; + bool RemoveAllImages() override; + + std::vector GetAllBackingImages(); + // Same as CreateBackingImage, but provides a progress notification. + bool CreateBackingImage(const std::string& name, uint64_t size, int flags, + std::function&& on_progress); + + // Returns true if the named partition exists. This does not check the + // consistency of the backing image/data file. + bool PartitionExists(const std::string& name); + + // Validates that all images still have pinned extents. This will be removed + // once b/134588268 is fixed. + bool Validate(); + + void set_partition_opener(std::unique_ptr&& opener); + + // Writes |bytes| zeros at the beginning of the passed image + bool ZeroFillNewImage(const std::string& name, uint64_t bytes); + + private: + ImageManager(const std::string& metadata_dir, const std::string& data_dir); + std::string GetImageHeaderPath(const std::string& name); + std::string GetStatusFilePath(const std::string& image_name); + bool MapWithLoopDevice(const std::string& name, const std::chrono::milliseconds& timeout_ms, + std::string* path); + bool MapWithLoopDeviceList(const std::vector& device_list, const std::string& name, + const std::chrono::milliseconds& timeout_ms, std::string* path); + bool MapWithDmLinear(const IPartitionOpener& opener, const std::string& name, + const std::chrono::milliseconds& timeout_ms, std::string* path); + bool UnmapImageDevice(const std::string& name, bool force); + + ImageManager(const ImageManager&) = delete; + ImageManager& operator=(const ImageManager&) = delete; + ImageManager& operator=(ImageManager&&) = delete; + ImageManager(ImageManager&&) = delete; + + std::string metadata_dir_; + std::string data_dir_; + std::unique_ptr partition_opener_; +}; + +// RAII helper class for mapping and opening devices with an ImageManager. +class MappedDevice final { + public: + static std::unique_ptr Open(IImageManager* manager, + const std::chrono::milliseconds& timeout_ms, + const std::string& name); + + ~MappedDevice(); + + int fd() const { return fd_; } + const std::string& path() const { return path_; } + + protected: + MappedDevice(IImageManager* manager, const std::string& name, const std::string& path); + + IImageManager* manager_; + std::string name_; + std::string path_; + android::base::unique_fd fd_; +}; + +} // namespace fiemap +} // namespace android diff --git a/fs_mgr/libfiemap/include/libfiemap/split_fiemap_writer.h b/fs_mgr/libfiemap/include/libfiemap/split_fiemap_writer.h new file mode 100644 index 000000000..feffb3d15 --- /dev/null +++ b/fs_mgr/libfiemap/include/libfiemap/split_fiemap_writer.h @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include "fiemap_writer.h" + +namespace android { +namespace fiemap { + +// Wrapper around FiemapWriter that is able to split images across files if +// necessary. +class SplitFiemap final { + public: + using ProgressCallback = std::function; + + // Create a new split fiemap file. If |max_piece_size| is 0, the number of + // pieces will be determined automatically by detecting the filesystem. + // Otherwise, the file will be split evenly (with the remainder in the + // final file). + static std::unique_ptr Create(const std::string& file_path, uint64_t file_size, + uint64_t max_piece_size, + ProgressCallback progress = {}); + + // Open an existing split fiemap file. + static std::unique_ptr Open(const std::string& file_path); + + ~SplitFiemap(); + + // Return a list of all files created for a split file. + static bool GetSplitFileList(const std::string& file_path, std::vector* list); + + // Destroy all components of a split file. If the root file does not exist, + // this returns true and does not report an error. + static bool RemoveSplitFiles(const std::string& file_path, std::string* message = nullptr); + + // Return whether all components of a split file still have pinned extents. + bool HasPinnedExtents() const; + + // Helper method for writing data that spans files. Note there is no seek + // method (yet); this starts at 0 and increments the position by |bytes|. + bool Write(const void* data, uint64_t bytes); + + // Flush all writes to all split files. + bool Flush(); + + const std::vector& extents(); + uint32_t block_size() const; + uint64_t size() const { return total_size_; } + const std::string& bdev_path() const; + + // Non-copyable & Non-movable + SplitFiemap(const SplitFiemap&) = delete; + SplitFiemap& operator=(const SplitFiemap&) = delete; + SplitFiemap& operator=(SplitFiemap&&) = delete; + SplitFiemap(SplitFiemap&&) = delete; + + private: + SplitFiemap() = default; + void AddFile(FiemapUniquePtr&& file); + + bool creating_ = false; + std::string list_file_; + std::vector files_; + std::vector extents_; + uint64_t total_size_ = 0; + + // Most recently open file and position for Write(). + size_t cursor_index_ = 0; + uint64_t cursor_file_pos_ = 0; + android::base::unique_fd cursor_fd_; +}; + +} // namespace fiemap +} // namespace android diff --git a/fs_mgr/libfiemap/metadata.cpp b/fs_mgr/libfiemap/metadata.cpp new file mode 100644 index 000000000..597efe987 --- /dev/null +++ b/fs_mgr/libfiemap/metadata.cpp @@ -0,0 +1,196 @@ +// +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "metadata.h" + +#include +#include +#include + +#include +#include +#include + +#include "utility.h" + +namespace android { +namespace fiemap { + +using namespace android::fs_mgr; + +static constexpr uint32_t kMaxMetadataSize = 256 * 1024; + +std::string GetMetadataFile(const std::string& metadata_dir) { + return JoinPaths(metadata_dir, "lp_metadata"); +} + +bool MetadataExists(const std::string& metadata_dir) { + auto metadata_file = GetMetadataFile(metadata_dir); + return access(metadata_file.c_str(), F_OK) == 0; +} + +std::unique_ptr OpenMetadata(const std::string& metadata_dir) { + auto metadata_file = GetMetadataFile(metadata_dir); + auto metadata = ReadFromImageFile(metadata_file); + if (!metadata) { + LOG(ERROR) << "Could not read metadata file " << metadata_file; + return nullptr; + } + return metadata; +} + +// :TODO: overwrite on create if open fails +std::unique_ptr OpenOrCreateMetadata(const std::string& metadata_dir, + SplitFiemap* file) { + auto metadata_file = GetMetadataFile(metadata_dir); + + PartitionOpener opener; + std::unique_ptr builder; + if (access(metadata_file.c_str(), R_OK)) { + if (errno != ENOENT) { + PLOG(ERROR) << "access " << metadata_file << " failed:"; + return nullptr; + } + + auto data_device = GetDevicePathForFile(file); + + BlockDeviceInfo device_info; + if (!opener.GetInfo(data_device, &device_info)) { + LOG(ERROR) << "Could not read partition: " << data_device; + return nullptr; + } + + std::vector block_devices = {device_info}; + auto super_name = android::base::Basename(data_device); + builder = MetadataBuilder::New(block_devices, super_name, kMaxMetadataSize, 1); + } else { + auto metadata = OpenMetadata(metadata_dir); + if (!metadata) { + return nullptr; + } + builder = MetadataBuilder::New(*metadata.get(), &opener); + } + + if (!builder) { + LOG(ERROR) << "Could not create metadata builder"; + return nullptr; + } + return builder; +} + +bool SaveMetadata(MetadataBuilder* builder, const std::string& metadata_dir) { + auto exported = builder->Export(); + if (!exported) { + LOG(ERROR) << "Unable to export new metadata"; + return false; + } + + // If there are no more partitions in the metadata, just delete the file. + auto metadata_file = GetMetadataFile(metadata_dir); + if (exported->partitions.empty() && android::base::RemoveFileIfExists(metadata_file)) { + return true; + } + if (!WriteToImageFile(metadata_file, *exported.get())) { + LOG(ERROR) << "Unable to save new metadata"; + return false; + } + return true; +} + +bool RemoveAllMetadata(const std::string& dir) { + auto metadata_file = GetMetadataFile(dir); + return android::base::RemoveFileIfExists(metadata_file); +} + +bool FillPartitionExtents(MetadataBuilder* builder, Partition* partition, SplitFiemap* file, + uint64_t partition_size) { + auto block_device = android::base::Basename(GetDevicePathForFile(file)); + + uint64_t sectors_needed = partition_size / LP_SECTOR_SIZE; + for (const auto& extent : file->extents()) { + if (extent.fe_length % LP_SECTOR_SIZE != 0) { + LOG(ERROR) << "Extent is not sector-aligned: " << extent.fe_length; + return false; + } + if (extent.fe_physical % LP_SECTOR_SIZE != 0) { + LOG(ERROR) << "Extent physical sector is not sector-aligned: " << extent.fe_physical; + return false; + } + + uint64_t num_sectors = + std::min(static_cast(extent.fe_length / LP_SECTOR_SIZE), sectors_needed); + if (!num_sectors || !sectors_needed) { + // This should never happen, but we include it just in case. It would + // indicate that the last filesystem block had multiple extents. + LOG(WARNING) << "FiemapWriter allocated extra blocks"; + break; + } + + uint64_t physical_sector = extent.fe_physical / LP_SECTOR_SIZE; + if (!builder->AddLinearExtent(partition, block_device, num_sectors, physical_sector)) { + LOG(ERROR) << "Could not add extent to lp metadata"; + return false; + } + + sectors_needed -= num_sectors; + } + return true; +} + +bool RemoveImageMetadata(const std::string& metadata_dir, const std::string& partition_name) { + if (!MetadataExists(metadata_dir)) { + return true; + } + auto metadata = OpenMetadata(metadata_dir); + if (!metadata) { + return false; + } + + PartitionOpener opener; + auto builder = MetadataBuilder::New(*metadata.get(), &opener); + if (!builder) { + return false; + } + builder->RemovePartition(partition_name); + return SaveMetadata(builder.get(), metadata_dir); +} + +bool UpdateMetadata(const std::string& metadata_dir, const std::string& partition_name, + SplitFiemap* file, uint64_t partition_size, bool readonly) { + auto builder = OpenOrCreateMetadata(metadata_dir, file); + if (!builder) { + return false; + } + auto partition = builder->FindPartition(partition_name); + if (!partition) { + int attrs = 0; + if (readonly) attrs |= LP_PARTITION_ATTR_READONLY; + + if ((partition = builder->AddPartition(partition_name, attrs)) == nullptr) { + LOG(ERROR) << "Could not add partition " << partition_name << " to metadata"; + return false; + } + } + partition->RemoveExtents(); + + if (!FillPartitionExtents(builder.get(), partition, file, partition_size)) { + return false; + } + return SaveMetadata(builder.get(), metadata_dir); +} + +} // namespace fiemap +} // namespace android diff --git a/fs_mgr/libfiemap/metadata.h b/fs_mgr/libfiemap/metadata.h new file mode 100644 index 000000000..f0ce23ef2 --- /dev/null +++ b/fs_mgr/libfiemap/metadata.h @@ -0,0 +1,36 @@ +// +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include +#include + +#include +#include + +namespace android { +namespace fiemap { + +bool MetadataExists(const std::string& metadata_dir); +std::unique_ptr OpenMetadata(const std::string& metadata_dir); +bool UpdateMetadata(const std::string& metadata_dir, const std::string& partition_name, + SplitFiemap* file, uint64_t partition_size, bool readonly); +bool RemoveImageMetadata(const std::string& metadata_dir, const std::string& partition_name); +bool RemoveAllMetadata(const std::string& dir); + +} // namespace fiemap +} // namespace android diff --git a/fs_mgr/libfiemap/passthrough.cpp b/fs_mgr/libfiemap/passthrough.cpp new file mode 100644 index 000000000..1ccd9a0c6 --- /dev/null +++ b/fs_mgr/libfiemap/passthrough.cpp @@ -0,0 +1,29 @@ +// +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +namespace android { +namespace fiemap { + +std::unique_ptr IImageManager::Open(const std::string& dir_prefix, + const std::chrono::milliseconds& timeout_ms) { + (void)timeout_ms; + return ImageManager::Open(dir_prefix); +} + +} // namespace fiemap +} // namespace android diff --git a/fs_mgr/libfiemap/split_fiemap_writer.cpp b/fs_mgr/libfiemap/split_fiemap_writer.cpp new file mode 100644 index 000000000..cc54f20e9 --- /dev/null +++ b/fs_mgr/libfiemap/split_fiemap_writer.cpp @@ -0,0 +1,303 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "utility.h" + +namespace android { +namespace fiemap { + +using android::base::unique_fd; + +// We use a four-digit suffix at the end of filenames. +static const size_t kMaxFilePieces = 500; + +std::unique_ptr SplitFiemap::Create(const std::string& file_path, uint64_t file_size, + uint64_t max_piece_size, + ProgressCallback progress) { + if (!file_size) { + LOG(ERROR) << "Cannot create a fiemap for a 0-length file: " << file_path; + return nullptr; + } + + if (!max_piece_size) { + max_piece_size = DetermineMaximumFileSize(file_path); + if (!max_piece_size) { + LOG(ERROR) << "Could not determine maximum file size for " << file_path; + return nullptr; + } + } + + // Remove any existing file. + RemoveSplitFiles(file_path); + + // Call |progress| only when the total percentage would significantly change. + int permille = -1; + uint64_t total_bytes_written = 0; + auto on_progress = [&](uint64_t written, uint64_t) -> bool { + uint64_t actual_written = total_bytes_written + written; + int new_permille = (actual_written * 1000) / file_size; + if (new_permille != permille && actual_written < file_size) { + if (progress && !progress(actual_written, file_size)) { + return false; + } + permille = new_permille; + } + return true; + }; + + std::unique_ptr out(new SplitFiemap()); + out->creating_ = true; + out->list_file_ = file_path; + + // Create the split files. + uint64_t remaining_bytes = file_size; + while (remaining_bytes) { + if (out->files_.size() >= kMaxFilePieces) { + LOG(ERROR) << "Requested size " << file_size << " created too many split files"; + return nullptr; + } + std::string chunk_path = + android::base::StringPrintf("%s.%04d", file_path.c_str(), (int)out->files_.size()); + uint64_t chunk_size = std::min(max_piece_size, remaining_bytes); + auto writer = FiemapWriter::Open(chunk_path, chunk_size, true, on_progress); + if (!writer) { + return nullptr; + } + + // To make sure the alignment doesn't create too much inconsistency, we + // account the *actual* size, not the requested size. + total_bytes_written += writer->size(); + + // writer->size() is block size aligned and could be bigger than remaining_bytes + // If remaining_bytes is bigger, set remaining_bytes to 0 to avoid underflow error. + remaining_bytes = remaining_bytes > writer->size() ? (remaining_bytes - writer->size()) : 0; + + out->AddFile(std::move(writer)); + } + + // Create the split file list. + unique_fd fd(open(out->list_file_.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0660)); + if (fd < 0) { + PLOG(ERROR) << "Failed to open " << file_path; + return nullptr; + } + + for (const auto& writer : out->files_) { + std::string line = android::base::Basename(writer->file_path()) + "\n"; + if (!android::base::WriteFully(fd, line.data(), line.size())) { + PLOG(ERROR) << "Write failed " << file_path; + return nullptr; + } + } + + // Unset this bit, so we don't unlink on destruction. + out->creating_ = false; + return out; +} + +std::unique_ptr SplitFiemap::Open(const std::string& file_path) { + std::vector files; + if (!GetSplitFileList(file_path, &files)) { + return nullptr; + } + + std::unique_ptr out(new SplitFiemap()); + out->list_file_ = file_path; + + for (const auto& file : files) { + auto writer = FiemapWriter::Open(file, 0, false); + if (!writer) { + // Error was logged in Open(). + return nullptr; + } + out->AddFile(std::move(writer)); + } + return out; +} + +bool SplitFiemap::GetSplitFileList(const std::string& file_path, std::vector* list) { + // This is not the most efficient thing, but it is simple and recovering + // the fiemap/fibmap is much more expensive. + std::string contents; + if (!android::base::ReadFileToString(file_path, &contents, true)) { + PLOG(ERROR) << "Error reading file: " << file_path; + return false; + } + + std::vector names = android::base::Split(contents, "\n"); + std::string dir = android::base::Dirname(file_path); + for (const auto& name : names) { + if (!name.empty()) { + list->emplace_back(dir + "/" + name); + } + } + return true; +} + +bool SplitFiemap::RemoveSplitFiles(const std::string& file_path, std::string* message) { + // Early exit if this does not exist, and do not report an error. + if (access(file_path.c_str(), F_OK) && errno == ENOENT) { + return true; + } + + bool ok = true; + std::vector files; + if (GetSplitFileList(file_path, &files)) { + for (const auto& file : files) { + ok &= android::base::RemoveFileIfExists(file, message); + } + } + ok &= android::base::RemoveFileIfExists(file_path, message); + return ok; +} + +bool SplitFiemap::HasPinnedExtents() const { + for (const auto& file : files_) { + if (!FiemapWriter::HasPinnedExtents(file->file_path())) { + return false; + } + } + return true; +} + +const std::vector& SplitFiemap::extents() { + if (extents_.empty()) { + for (const auto& file : files_) { + const auto& extents = file->extents(); + extents_.insert(extents_.end(), extents.begin(), extents.end()); + } + } + return extents_; +} + +bool SplitFiemap::Write(const void* data, uint64_t bytes) { + // Open the current file. + FiemapWriter* file = files_[cursor_index_].get(); + + const uint8_t* data_ptr = reinterpret_cast(data); + uint64_t bytes_remaining = bytes; + while (bytes_remaining) { + // How many bytes can we write into the current file? + uint64_t file_bytes_left = file->size() - cursor_file_pos_; + if (!file_bytes_left) { + if (cursor_index_ == files_.size() - 1) { + LOG(ERROR) << "write past end of file requested"; + return false; + } + + // No space left in the current file, but we have more files to + // use, so prep the next one. + cursor_fd_ = {}; + cursor_file_pos_ = 0; + file = files_[++cursor_index_].get(); + file_bytes_left = file->size(); + } + + // Open the current file if it's not open. + if (cursor_fd_ < 0) { + cursor_fd_.reset(open(file->file_path().c_str(), O_CLOEXEC | O_WRONLY)); + if (cursor_fd_ < 0) { + PLOG(ERROR) << "open failed: " << file->file_path(); + return false; + } + CHECK(cursor_file_pos_ == 0); + } + + if (!FiemapWriter::HasPinnedExtents(file->file_path())) { + LOG(ERROR) << "file is no longer pinned: " << file->file_path(); + return false; + } + + uint64_t bytes_to_write = std::min(file_bytes_left, bytes_remaining); + if (!android::base::WriteFully(cursor_fd_, data_ptr, bytes_to_write)) { + PLOG(ERROR) << "write failed: " << file->file_path(); + return false; + } + data_ptr += bytes_to_write; + bytes_remaining -= bytes_to_write; + cursor_file_pos_ += bytes_to_write; + } + + // If we've reached the end of the current file, close it for sanity. + if (cursor_file_pos_ == file->size()) { + cursor_fd_ = {}; + } + return true; +} + +bool SplitFiemap::Flush() { + for (const auto& file : files_) { + unique_fd fd(open(file->file_path().c_str(), O_RDONLY | O_CLOEXEC)); + if (fd < 0) { + PLOG(ERROR) << "open failed: " << file->file_path(); + return false; + } + if (fsync(fd)) { + PLOG(ERROR) << "fsync failed: " << file->file_path(); + return false; + } + } + return true; +} + +SplitFiemap::~SplitFiemap() { + if (!creating_) { + return; + } + + // We failed to finish creating, so unlink everything. + unlink(list_file_.c_str()); + for (auto&& file : files_) { + std::string path = file->file_path(); + file = nullptr; + + unlink(path.c_str()); + } +} + +void SplitFiemap::AddFile(FiemapUniquePtr&& file) { + total_size_ += file->size(); + files_.emplace_back(std::move(file)); +} + +uint32_t SplitFiemap::block_size() const { + return files_[0]->block_size(); +} + +const std::string& SplitFiemap::bdev_path() const { + return files_[0]->bdev_path(); +} + +} // namespace fiemap +} // namespace android diff --git a/fs_mgr/libfiemap/testdata/file_32k b/fs_mgr/libfiemap/testdata/file_32k new file mode 100644 index 000000000..12f3be4dd Binary files /dev/null and b/fs_mgr/libfiemap/testdata/file_32k differ diff --git a/fs_mgr/libfiemap/testdata/file_4k b/fs_mgr/libfiemap/testdata/file_4k new file mode 100644 index 000000000..08e7df176 Binary files /dev/null and b/fs_mgr/libfiemap/testdata/file_4k differ diff --git a/fs_mgr/libfiemap/testdata/unaligned_file b/fs_mgr/libfiemap/testdata/unaligned_file new file mode 100644 index 000000000..c107c2637 Binary files /dev/null and b/fs_mgr/libfiemap/testdata/unaligned_file differ diff --git a/fs_mgr/libfiemap/utility.cpp b/fs_mgr/libfiemap/utility.cpp new file mode 100644 index 000000000..955e544fc --- /dev/null +++ b/fs_mgr/libfiemap/utility.cpp @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utility.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace android { +namespace fiemap { + +using namespace std::string_literals; +using android::base::unique_fd; + +static constexpr char kUserdataDevice[] = "/dev/block/by-name/userdata"; + +uint64_t DetermineMaximumFileSize(const std::string& file_path) { + // Create the smallest file possible (one block). + auto writer = FiemapWriter::Open(file_path, 1); + if (!writer) { + return 0; + } + + uint64_t result = 0; + switch (writer->fs_type()) { + case EXT4_SUPER_MAGIC: + // The minimum is 16GiB, so just report that. If we wanted we could parse the + // superblock and figure out if 64-bit support is enabled. + result = 17179869184ULL; + break; + case F2FS_SUPER_MAGIC: + // Formula is from https://www.kernel.org/doc/Documentation/filesystems/f2fs.txt + // 4KB * (923 + 2 * 1018 + 2 * 1018 * 1018 + 1018 * 1018 * 1018) := 3.94TB. + result = 4329690886144ULL; + break; + case MSDOS_SUPER_MAGIC: + // 4GB-1, which we want aligned to the block size. + result = 4294967295; + result -= (result % writer->block_size()); + break; + default: + LOG(ERROR) << "Unknown file system type: " << writer->fs_type(); + break; + } + + // Close and delete the temporary file. + writer = nullptr; + unlink(file_path.c_str()); + + return result; +} + +// Given a SplitFiemap, this returns a device path that will work during first- +// stage init (i.e., its path can be found by InitRequiredDevices). +std::string GetDevicePathForFile(SplitFiemap* file) { + auto bdev_path = file->bdev_path(); + + struct stat userdata, given; + if (!stat(bdev_path.c_str(), &given) && !stat(kUserdataDevice, &userdata)) { + if (S_ISBLK(given.st_mode) && S_ISBLK(userdata.st_mode) && + given.st_rdev == userdata.st_rdev) { + return kUserdataDevice; + } + } + return bdev_path; +} + +std::string JoinPaths(const std::string& dir, const std::string& file) { + if (android::base::EndsWith(dir, "/")) { + return dir + file; + } + return dir + "/" + file; +} + +bool F2fsPinBeforeAllocate(int file_fd, bool* supported) { + struct stat st; + if (fstat(file_fd, &st) < 0) { + PLOG(ERROR) << "stat failed"; + return false; + } + std::string bdev; + if (!BlockDeviceToName(major(st.st_dev), minor(st.st_dev), &bdev)) { + LOG(ERROR) << "Failed to get block device name for " << major(st.st_dev) << ":" + << minor(st.st_dev); + return false; + } + + std::string contents; + std::string feature_file = "/sys/fs/f2fs/" + bdev + "/features"; + if (!android::base::ReadFileToString(feature_file, &contents)) { + PLOG(ERROR) << "read failed: " << feature_file; + return false; + } + contents = android::base::Trim(contents); + + auto features = android::base::Split(contents, ", "); + auto iter = std::find(features.begin(), features.end(), "pin_file"s); + *supported = (iter != features.end()); + return true; +} + +bool BlockDeviceToName(uint32_t major, uint32_t minor, std::string* bdev_name) { + // The symlinks in /sys/dev/block point to the block device node under /sys/device/.. + // The directory name in the target corresponds to the name of the block device. We use + // that to extract the block device name. + // e.g for block device name 'ram0', there exists a symlink named '1:0' in /sys/dev/block as + // follows. + // 1:0 -> ../../devices/virtual/block/ram0 + std::string sysfs_path = ::android::base::StringPrintf("/sys/dev/block/%u:%u", major, minor); + std::string sysfs_bdev; + + if (!::android::base::Readlink(sysfs_path, &sysfs_bdev)) { + PLOG(ERROR) << "Failed to read link at: " << sysfs_path; + return false; + } + + *bdev_name = ::android::base::Basename(sysfs_bdev); + // Paranoid sanity check to make sure we just didn't get the + // input in return as-is. + if (sysfs_bdev == *bdev_name) { + LOG(ERROR) << "Malformed symlink for block device: " << sysfs_bdev; + return false; + } + + return true; +} + +bool FilesystemHasReliablePinning(const std::string& file, bool* supported) { + struct statfs64 sfs; + if (statfs64(file.c_str(), &sfs)) { + PLOG(ERROR) << "statfs failed: " << file; + return false; + } + if (sfs.f_type != F2FS_SUPER_MAGIC) { + *supported = true; + return true; + } + + unique_fd fd(open(file.c_str(), O_RDONLY | O_CLOEXEC)); + if (fd < 0) { + PLOG(ERROR) << "open failed: " << file; + return false; + } + return F2fsPinBeforeAllocate(fd, supported); +} + +} // namespace fiemap +} // namespace android diff --git a/fs_mgr/libfiemap/utility.h b/fs_mgr/libfiemap/utility.h new file mode 100644 index 000000000..24ebc578d --- /dev/null +++ b/fs_mgr/libfiemap/utility.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include + +#include + +namespace android { +namespace fiemap { + +// Given a file that will be created, determine the maximum size its containing +// filesystem allows. Note this is a theoretical maximum size; free space is +// ignored entirely. +uint64_t DetermineMaximumFileSize(const std::string& file_path); + +// Given a SplitFiemap, this returns a device path that will work during first- +// stage init (i.e., its path can be found by InitRequiredDevices). +std::string GetDevicePathForFile(android::fiemap::SplitFiemap* file); + +// Combine two path components into a single path. +std::string JoinPaths(const std::string& dir, const std::string& file); + +// Given a file within an F2FS filesystem, return whether or not the filesystem +// supports the "pin_file" feature, which requires pinning before fallocation. +bool F2fsPinBeforeAllocate(int file_fd, bool* supported); + +// Given a major/minor device number, return its canonical name such that +// /dev/block/ resolves to the device. +bool BlockDeviceToName(uint32_t major, uint32_t minor, std::string* bdev_name); + +// This is the same as F2fsPinBeforeAllocate, however, it will return true +// (and supported = true) for non-f2fs filesystems. It is intended to be used +// in conjunction with ImageManager to reject image requests for reliable use +// cases (such as snapshots or adb remount). +bool FilesystemHasReliablePinning(const std::string& file, bool* supported); + +} // namespace fiemap +} // namespace android