openkylin
/
platform_system_core
mirror of https://gitee.com/openkylin/platform_system_core.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge "Remove libfiemap_writer from fs_mgr and move it to gsid." 

am: 3d5b8bc373

Change-Id: I8950fcf3b780da4e00f813b90e6689d0a164141e
This commit is contained in:
David Anderson 2019-07-15 16:22:42 -07:00 committed by android-build-merger
parent 61c7cc6d88 3d5b8bc373
commit 759f119fc9
14 changed files with 0 additions and 2011 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fs_mgr/libfiemap_writer/.clang-format
View File

@ -1 +0,0 @@
../../.clang-format-4

64
fs_mgr/libfiemap_writer/Android.bp
View File

@ -1,64 +0,0 @@
//
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
cc_library_static {
name: "libfiemap_writer",
defaults: ["fs_mgr_defaults"],
recovery_available: true,
export_include_dirs: ["include"],
cflags: [
"-D_FILE_OFFSET_BITS=64",
],
srcs: [
"fiemap_writer.cpp",
"split_fiemap_writer.cpp",
"utility.cpp",
],
static_libs: [
"libdm",
"libext4_utils",
],
header_libs: [
"libbase_headers",
"liblog_headers",
],
}
cc_test {
name: "fiemap_writer_test",
cflags: [
"-D_FILE_OFFSET_BITS=64",
],
static_libs: [
"libbase",
"libdm",
"libfiemap_writer",
"liblog",
],
data: [
"testdata/unaligned_file",
"testdata/file_4k",
"testdata/file_32k",
],
srcs: [
"fiemap_writer_test.cpp",
],
}

22
fs_mgr/libfiemap_writer/Android.mk
View File

@ -1,22 +0,0 @@
#
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := VtsFiemapWriterTest
-include test/vts/tools/build/Android.host_config.mk

29
fs_mgr/libfiemap_writer/AndroidTest.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2019 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration description="Config for VTS VtsFiemapWriterTest">
<option name="config-descriptor:metadata" key="plan" value="vts-kernel" />
<target_preparer class="com.android.compatibility.common.tradefed.targetprep.VtsFilePusher">
<option name="abort-on-push-failure" value="false"/>
<option name="push-group" value="HostDrivenTest.push"/>
</target_preparer>
<test class="com.android.tradefed.testtype.VtsMultiDeviceTest">
<option name="test-module-name" value="VtsFiemapWriterTest"/>
<option name="binary-test-source" value="_32bit::DATA/nativetest/fiemap_writer_test/fiemap_writer_test" />
<option name="binary-test-source" value="_64bit::DATA/nativetest64/fiemap_writer_test/fiemap_writer_test" />
<option name="binary-test-type" value="gtest"/>
<option name="test-timeout" value="1m"/>
</test>
</configuration>

745
fs_mgr/libfiemap_writer/fiemap_writer.cpp
View File

@ -1,745 +0,0 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <libfiemap_writer/fiemap_writer.h>
#include <dirent.h>
#include <fcntl.h>
#include <linux/fs.h>
#include <stdio.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/sysmacros.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>
#include <limits>
#include <string>
#include <utility>
#include <vector>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include <libdm/dm.h>
namespace android {
namespace fiemap_writer {
using namespace android::dm;
// We are expecting no more than 512 extents in a fiemap of the file we create.
// If we find more, then it is treated as error for now.
static constexpr const uint32_t kMaxExtents = 512;
// TODO: Fallback to using fibmap if FIEMAP_EXTENT_MERGED is set.
static constexpr const uint32_t kUnsupportedExtentFlags =
FIEMAP_EXTENT_UNKNOWN | FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_DELALLOC |
FIEMAP_EXTENT_NOT_ALIGNED | FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_DATA_TAIL |
FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_SHARED | FIEMAP_EXTENT_MERGED;
// Large file support must be enabled.
static_assert(sizeof(off_t) == sizeof(uint64_t));
static inline void cleanup(const std::string& file_path, bool created) {
if (created) {
unlink(file_path.c_str());
}
}
static bool BlockDeviceToName(uint32_t major, uint32_t minor, std::string* bdev_name) {
// The symlinks in /sys/dev/block point to the block device node under /sys/device/..
// The directory name in the target corresponds to the name of the block device. We use
// that to extract the block device name.
// e.g for block device name 'ram0', there exists a symlink named '1:0' in /sys/dev/block as
// follows.
// 1:0 -> ../../devices/virtual/block/ram0
std::string sysfs_path = ::android::base::StringPrintf("/sys/dev/block/%u:%u", major, minor);
std::string sysfs_bdev;
if (!::android::base::Readlink(sysfs_path, &sysfs_bdev)) {
PLOG(ERROR) << "Failed to read link at: " << sysfs_path;
return false;
}
*bdev_name = ::android::base::Basename(sysfs_bdev);
// Paranoid sanity check to make sure we just didn't get the
// input in return as-is.
if (sysfs_bdev == *bdev_name) {
LOG(ERROR) << "Malformed symlink for block device: " << sysfs_bdev;
return false;
}
return true;
}
static bool ValidateDmTarget(const DeviceMapper::TargetInfo& target) {
const auto& entry = target.spec;
if (entry.sector_start != 0) {
LOG(INFO) << "Stopping at target with non-zero starting sector";
return false;
}
auto target_type = DeviceMapper::GetTargetType(entry);
if (target_type == "bow" || target_type == "default-key" || target_type == "crypt") {
return true;
}
if (target_type == "linear") {
auto pieces = android::base::Split(target.data, " ");
if (pieces[1] != "0") {
LOG(INFO) << "Stopping at complex linear target with non-zero starting sector: "
<< pieces[1];
return false;
}
return true;
}
LOG(INFO) << "Stopping at complex target type " << target_type;
return false;
}
static bool DeviceMapperStackPop(const std::string& bdev, std::string* bdev_raw) {
*bdev_raw = bdev;
if (!::android::base::StartsWith(bdev, "dm-")) {
// We are at the bottom of the device mapper stack.
return true;
}
// Get the device name.
auto dm_name_file = "/sys/block/" + bdev + "/dm/name";
std::string dm_name;
if (!android::base::ReadFileToString(dm_name_file, &dm_name)) {
PLOG(ERROR) << "Could not read file: " << dm_name_file;
return false;
}
dm_name = android::base::Trim(dm_name);
auto& dm = DeviceMapper::Instance();
std::vector<DeviceMapper::TargetInfo> table;
if (!dm.GetTableInfo(dm_name, &table)) {
LOG(ERROR) << "Could not read device-mapper table for " << dm_name << " at " << bdev;
return false;
}
// The purpose of libfiemap_writer is to provide an extent-based view into
// a file. This is difficult if devices are not layered in a 1:1 manner;
// we would have to translate and break up extents based on the actual
// block mapping. Since this is too complex, we simply stop processing
// the device-mapper stack if we encounter a complex case.
//
// It is up to the caller to decide whether stopping at a virtual block
// device is allowable. In most cases it is not, because we want either
// "userdata" or an external volume. It is useful for tests however.
// Callers can check by comparing the device number to that of userdata,
// or by checking whether is a device-mapper node.
if (table.size() > 1) {
LOG(INFO) << "Stopping at complex table for " << dm_name << " at " << bdev;
return true;
}
if (!ValidateDmTarget(table[0])) {
return true;
}
auto dm_leaf_dir = "/sys/block/" + bdev + "/slaves";
auto d = std::unique_ptr<DIR, decltype(&closedir)>(opendir(dm_leaf_dir.c_str()), closedir);
if (d == nullptr) {
PLOG(ERROR) << "Failed to open: " << dm_leaf_dir;
return false;
}
struct dirent* de;
uint32_t num_leaves = 0;
std::string bdev_next = "";
while ((de = readdir(d.get())) != nullptr) {
if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) {
continue;
}
// We set the first name we find here
if (bdev_next.empty()) {
bdev_next = de->d_name;
}
num_leaves++;
}
// if we have more than one leaves, we return immediately. We can't continue to create the
// file since we don't know how to write it out using fiemap, so it will be readable via the
// underlying block devices later. The reader will also have to construct the same device mapper
// target in order read the file out.
if (num_leaves > 1) {
LOG(ERROR) << "Found " << num_leaves << " leaf block devices under device mapper device "
<< bdev;
return false;
}
// recursively call with the block device we found in order to pop the device mapper stack.
return DeviceMapperStackPop(bdev_next, bdev_raw);
}
bool FiemapWriter::GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path,
bool* uses_dm) {
struct stat sb;
if (stat(file_path.c_str(), &sb)) {
PLOG(ERROR) << "Failed to get stat for: " << file_path;
return false;
}
std::string bdev;
if (!BlockDeviceToName(major(sb.st_dev), minor(sb.st_dev), &bdev)) {
LOG(ERROR) << "Failed to get block device name for " << major(sb.st_dev) << ":"
<< minor(sb.st_dev);
return false;
}
std::string bdev_raw;
if (!DeviceMapperStackPop(bdev, &bdev_raw)) {
LOG(ERROR) << "Failed to get the bottom of the device mapper stack for device: " << bdev;
return false;
}
if (uses_dm) {
*uses_dm = (bdev_raw != bdev);
}
LOG(DEBUG) << "Popped device (" << bdev_raw << ") from device mapper stack starting with ("
<< bdev << ")";
*bdev_path = ::android::base::StringPrintf("/dev/block/%s", bdev_raw.c_str());
// Make sure we are talking to a block device before calling it a success.
if (stat(bdev_path->c_str(), &sb)) {
PLOG(ERROR) << "Failed to get stat for block device: " << *bdev_path;
return false;
}
if ((sb.st_mode & S_IFMT) != S_IFBLK) {
PLOG(ERROR) << "File: " << *bdev_path << " is not a block device";
return false;
}
return true;
}
static bool GetBlockDeviceSize(int bdev_fd, const std::string& bdev_path, uint64_t* bdev_size) {
uint64_t size_in_bytes = 0;
if (ioctl(bdev_fd, BLKGETSIZE64, &size_in_bytes)) {
PLOG(ERROR) << "Failed to get total size for: " << bdev_path;
return false;
}
*bdev_size = size_in_bytes;
return true;
}
static uint64_t GetFileSize(const std::string& file_path) {
struct stat sb;
if (stat(file_path.c_str(), &sb)) {
PLOG(ERROR) << "Failed to get size for file: " << file_path;
return 0;
}
return sb.st_size;
}
static bool PerformFileChecks(const std::string& file_path, uint64_t file_size, uint64_t* blocksz,
uint32_t* fs_type) {
struct statfs64 sfs;
if (statfs64(file_path.c_str(), &sfs)) {
PLOG(ERROR) << "Failed to read file system status at: " << file_path;
return false;
}
if (!sfs.f_bsize) {
LOG(ERROR) << "Unsupported block size: " << sfs.f_bsize;
return false;
}
// Check if the filesystem is of supported types.
// Only ext4, f2fs, and vfat are tested and supported.
switch (sfs.f_type) {
case EXT4_SUPER_MAGIC:
case F2FS_SUPER_MAGIC:
case MSDOS_SUPER_MAGIC:
break;
default:
LOG(ERROR) << "Unsupported file system type: 0x" << std::hex << sfs.f_type;
return false;
}
uint64_t available_bytes = sfs.f_bsize * sfs.f_bavail;
if (available_bytes <= file_size) {
LOG(ERROR) << "Not enough free space in file system to create file of size : " << file_size;
return false;
}
*blocksz = sfs.f_bsize;
*fs_type = sfs.f_type;
return true;
}
static bool FallocateFallback(int file_fd, uint64_t block_size, uint64_t file_size,
const std::string& file_path,
const std::function<bool(uint64_t, uint64_t)>& on_progress) {
// Even though this is much faster than writing zeroes, it is still slow
// enough that we need to fire the progress callback periodically. To
// easily achieve this, we seek in chunks. We use 1000 chunks since
// normally we only fire the callback on 1/1000th increments.
uint64_t bytes_per_chunk = std::max(file_size / 1000, block_size);
// Seek just to the end of each chunk and write a single byte, causing
// the filesystem to allocate blocks.
off_t cursor = 0;
off_t end = static_cast<off_t>(file_size);
while (cursor < end) {
cursor = std::min(static_cast<off_t>(cursor + bytes_per_chunk), end);
auto rv = TEMP_FAILURE_RETRY(lseek(file_fd, cursor - 1, SEEK_SET));
if (rv < 0) {
PLOG(ERROR) << "Failed to lseek " << file_path;
return false;
}
if (rv != cursor - 1) {
LOG(ERROR) << "Seek returned wrong offset " << rv << " for file " << file_path;
return false;
}
char buffer[] = {0};
if (!android::base::WriteFully(file_fd, buffer, 1)) {
PLOG(ERROR) << "Write failed: " << file_path;
return false;
}
if (on_progress && !on_progress(cursor, file_size)) {
return false;
}
}
return true;
}
static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blocksz,
uint64_t file_size, unsigned int fs_type,
std::function<bool(uint64_t, uint64_t)> on_progress) {
// Reserve space for the file on the file system and write it out to make sure the extents
// don't come back unwritten. Return from this function with the kernel file offset set to 0.
// If the filesystem is f2fs, then we also PIN the file on disk to make sure the blocks
// aren't moved around.
switch (fs_type) {
case EXT4_SUPER_MAGIC:
case F2FS_SUPER_MAGIC:
if (fallocate(file_fd, FALLOC_FL_ZERO_RANGE, 0, file_size)) {
PLOG(ERROR) << "Failed to allocate space for file: " << file_path
<< " size: " << file_size;
return false;
}
break;
case MSDOS_SUPER_MAGIC:
// fallocate() is not supported, and not needed, since VFAT does not support holes.
// Instead we can perform a much faster allocation.
return FallocateFallback(file_fd, blocksz, file_size, file_path, on_progress);
default:
LOG(ERROR) << "Missing fallocate() support for file system " << fs_type;
return false;
}
// write zeroes in 'blocksz' byte increments until we reach file_size to make sure the data
// blocks are actually written to by the file system and thus getting rid of the holes in the
// file.
auto buffer = std::unique_ptr<void, decltype(&free)>(calloc(1, blocksz), free);
if (buffer == nullptr) {
LOG(ERROR) << "failed to allocate memory for writing file";
return false;
}
off64_t offset = lseek64(file_fd, 0, SEEK_SET);
if (offset < 0) {
PLOG(ERROR) << "Failed to seek at the beginning of : " << file_path;
return false;
}
int permille = -1;
while (offset < file_size) {
if (!::android::base::WriteFully(file_fd, buffer.get(), blocksz)) {
PLOG(ERROR) << "Failed to write" << blocksz << " bytes at offset" << offset
<< " in file " << file_path;
return false;
}
offset += blocksz;
// Don't invoke the callback every iteration - wait until a significant
// chunk (here, 1/1000th) of the data has been processed.
int new_permille = (static_cast<uint64_t>(offset) * 1000) / file_size;
if (new_permille != permille && static_cast<uint64_t>(offset) != file_size) {
if (on_progress && !on_progress(offset, file_size)) {
return false;
}
permille = new_permille;
}
}
if (lseek64(file_fd, 0, SEEK_SET) < 0) {
PLOG(ERROR) << "Failed to reset offset at the beginning of : " << file_path;
return false;
}
// flush all writes here ..
if (fsync(file_fd)) {
PLOG(ERROR) << "Failed to synchronize written file:" << file_path;
return false;
}
// Send one last progress notification.
if (on_progress && !on_progress(file_size, file_size)) {
return false;
}
return true;
}
static bool PinFile(int file_fd, const std::string& file_path, uint32_t fs_type) {
if (fs_type != F2FS_SUPER_MAGIC) {
// No pinning necessary for ext4/msdos. The blocks, once allocated, are
// expected to be fixed.
return true;
}
// F2FS-specific ioctl
// It requires the below kernel commit merged in v4.16-rc1.
// 1ad71a27124c ("f2fs: add an ioctl to disable GC for specific file")
// In android-4.4,
// 56ee1e817908 ("f2fs: updates on v4.16-rc1")
// In android-4.9,
// 2f17e34672a8 ("f2fs: updates on v4.16-rc1")
// In android-4.14,
// ce767d9a55bc ("f2fs: updates on v4.16-rc1")
#ifndef F2FS_IOC_SET_PIN_FILE
#ifndef F2FS_IOCTL_MAGIC
#define F2FS_IOCTL_MAGIC 0xf5
#endif
#define F2FS_IOC_SET_PIN_FILE _IOW(F2FS_IOCTL_MAGIC, 13, __u32)
#endif
uint32_t pin_status = 1;
int error = ioctl(file_fd, F2FS_IOC_SET_PIN_FILE, &pin_status);
if (error < 0) {
if ((errno == ENOTTY) || (errno == ENOTSUP)) {
PLOG(ERROR) << "Failed to pin file, not supported by kernel: " << file_path;
} else {
PLOG(ERROR) << "Failed to pin file: " << file_path;
}
return false;
}
return true;
}
static bool IsFilePinned(int file_fd, const std::string& file_path, uint32_t fs_type) {
if (fs_type != F2FS_SUPER_MAGIC) {
// No pinning necessary for ext4 or vfat. The blocks, once allocated,
// are expected to be fixed.
return true;
}
// F2FS-specific ioctl
// It requires the below kernel commit merged in v4.16-rc1.
// 1ad71a27124c ("f2fs: add an ioctl to disable GC for specific file")
// In android-4.4,
// 56ee1e817908 ("f2fs: updates on v4.16-rc1")
// In android-4.9,
// 2f17e34672a8 ("f2fs: updates on v4.16-rc1")
// In android-4.14,
// ce767d9a55bc ("f2fs: updates on v4.16-rc1")
#ifndef F2FS_IOC_GET_PIN_FILE
#ifndef F2FS_IOCTL_MAGIC
#define F2FS_IOCTL_MAGIC 0xf5
#endif
#define F2FS_IOC_GET_PIN_FILE _IOR(F2FS_IOCTL_MAGIC, 14, __u32)
#endif
// f2fs: export FS_NOCOW_FL flag to user
uint32_t flags;
int error = ioctl(file_fd, FS_IOC_GETFLAGS, &flags);
if (error < 0) {
if ((errno == ENOTTY) || (errno == ENOTSUP)) {
PLOG(ERROR) << "Failed to get flags, not supported by kernel: " << file_path;
} else {
PLOG(ERROR) << "Failed to get flags: " << file_path;
}
return false;
}
if (!(flags & FS_NOCOW_FL)) {
LOG(ERROR) << "It is not pinned: " << file_path;
return false;
}
// F2FS_IOC_GET_PIN_FILE returns the number of blocks moved.
uint32_t moved_blocks_nr;
error = ioctl(file_fd, F2FS_IOC_GET_PIN_FILE, &moved_blocks_nr);
if (error < 0) {
if ((errno == ENOTTY) || (errno == ENOTSUP)) {
PLOG(ERROR) << "Failed to get file pin status, not supported by kernel: " << file_path;
} else {
PLOG(ERROR) << "Failed to get file pin status: " << file_path;
}
return false;
}
if (moved_blocks_nr) {
LOG(ERROR) << moved_blocks_nr << " blocks moved in file " << file_path;
}
return moved_blocks_nr == 0;
}
bool FiemapWriter::HasPinnedExtents(const std::string& file_path) {
android::base::unique_fd fd(open(file_path.c_str(), O_NOFOLLOW | O_CLOEXEC | O_RDONLY));
if (fd < 0) {
PLOG(ERROR) << "open: " << file_path;
return false;
}
struct statfs64 sfs;
if (fstatfs64(fd, &sfs)) {
PLOG(ERROR) << "fstatfs64: " << file_path;
return false;
}
return IsFilePinned(fd, file_path, sfs.f_type);
}
static bool ReadFiemap(int file_fd, const std::string& file_path,
std::vector<struct fiemap_extent>* extents) {
uint64_t fiemap_size =
sizeof(struct fiemap_extent) + kMaxExtents * sizeof(struct fiemap_extent);
auto buffer = std::unique_ptr<void, decltype(&free)>(calloc(1, fiemap_size), free);
if (buffer == nullptr) {
LOG(ERROR) << "Failed to allocate memory for fiemap";
return false;
}
struct fiemap* fiemap = reinterpret_cast<struct fiemap*>(buffer.get());
fiemap->fm_start = 0;
fiemap->fm_length = UINT64_MAX;
// make sure file is synced to disk before we read the fiemap
fiemap->fm_flags = FIEMAP_FLAG_SYNC;
fiemap->fm_extent_count = kMaxExtents;
if (ioctl(file_fd, FS_IOC_FIEMAP, fiemap)) {
PLOG(ERROR) << "Failed to get FIEMAP from the kernel for file: " << file_path;
return false;
}
if (fiemap->fm_mapped_extents == 0) {
LOG(ERROR) << "File " << file_path << " has zero extents";
return false;
}
// Iterate through each extent read and make sure its valid before adding it to the vector
bool last_extent_seen = false;
struct fiemap_extent* extent = &fiemap->fm_extents[0];
for (uint32_t i = 0; i < fiemap->fm_mapped_extents; i++, extent++) {
// LogExtent(i + 1, *extent);
if (extent->fe_flags & kUnsupportedExtentFlags) {
LOG(ERROR) << "Extent " << i + 1 << " of file " << file_path
<< " has unsupported flags";
extents->clear();
return false;
}
if (extent->fe_flags & FIEMAP_EXTENT_LAST) {
last_extent_seen = true;
if (i != (fiemap->fm_mapped_extents - 1)) {
LOG(WARNING) << "Extents are being received out-of-order";
}
}
extents->emplace_back(std::move(*extent));
}
if (!last_extent_seen) {
// The file is possibly too fragmented.
if (fiemap->fm_mapped_extents == kMaxExtents) {
LOG(ERROR) << "File is too fragmented, needs more than " << kMaxExtents << " extents.";
}
extents->clear();
}
return last_extent_seen;
}
static bool ReadFibmap(int file_fd, const std::string& file_path,
std::vector<struct fiemap_extent>* extents) {
struct stat s;
if (fstat(file_fd, &s)) {
PLOG(ERROR) << "Failed to stat " << file_path;
return false;
}
unsigned int blksize;
if (ioctl(file_fd, FIGETBSZ, &blksize) < 0) {
PLOG(ERROR) << "Failed to get FIGETBSZ for " << file_path;
return false;
}
if (!blksize) {
LOG(ERROR) << "Invalid filesystem block size: " << blksize;
return false;
}
uint64_t num_blocks = (s.st_size + blksize - 1) / blksize;
if (num_blocks > std::numeric_limits<uint32_t>::max()) {
LOG(ERROR) << "Too many blocks for FIBMAP (" << num_blocks << ")";
return false;
}
for (uint32_t last_block, block_number = 0; block_number < num_blocks; block_number++) {
uint32_t block = block_number;
if (ioctl(file_fd, FIBMAP, &block)) {
PLOG(ERROR) << "Failed to get FIBMAP for file " << file_path;
return false;
}
if (!block) {
LOG(ERROR) << "Logical block " << block_number << " is a hole, which is not supported";
return false;
}
if (!extents->empty() && block == last_block + 1) {
extents->back().fe_length += blksize;
} else {
extents->push_back(fiemap_extent{.fe_logical = block_number,
.fe_physical = static_cast<uint64_t>(block) * blksize,
.fe_length = static_cast<uint64_t>(blksize),
.fe_flags = 0});
}
last_block = block;
}
return true;
}
FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_size, bool create,
std::function<bool(uint64_t, uint64_t)> progress) {
// if 'create' is false, open an existing file and do not truncate.
int open_flags = O_RDWR | O_CLOEXEC;
if (create) {
if (access(file_path.c_str(), F_OK) == 0) {
LOG(WARNING) << "File " << file_path << " already exists, truncating";
}
open_flags |= O_CREAT | O_TRUNC;
}
::android::base::unique_fd file_fd(
TEMP_FAILURE_RETRY(open(file_path.c_str(), open_flags, S_IRUSR | S_IWUSR)));
if (file_fd < 0) {
PLOG(ERROR) << "Failed to create file at: " << file_path;
return nullptr;
}
std::string abs_path;
if (!::android::base::Realpath(file_path, &abs_path)) {
PLOG(ERROR) << "Invalid file path: " << file_path;
cleanup(file_path, create);
return nullptr;
}
std::string bdev_path;
if (!GetBlockDeviceForFile(abs_path, &bdev_path)) {
LOG(ERROR) << "Failed to get block dev path for file: " << file_path;
cleanup(abs_path, create);
return nullptr;
}
::android::base::unique_fd bdev_fd(
TEMP_FAILURE_RETRY(open(bdev_path.c_str(), O_RDONLY | O_CLOEXEC)));
if (bdev_fd < 0) {
PLOG(ERROR) << "Failed to open block device: " << bdev_path;
cleanup(file_path, create);
return nullptr;
}
uint64_t bdevsz;
if (!GetBlockDeviceSize(bdev_fd, bdev_path, &bdevsz)) {
LOG(ERROR) << "Failed to get block device size for : " << bdev_path;
cleanup(file_path, create);
return nullptr;
}
if (!create) {
file_size = GetFileSize(abs_path);
if (file_size == 0) {
LOG(ERROR) << "Invalid file size of zero bytes for file: " << abs_path;
return nullptr;
}
}
uint64_t blocksz;
uint32_t fs_type;
if (!PerformFileChecks(abs_path, file_size, &blocksz, &fs_type)) {
LOG(ERROR) << "Failed to validate file or file system for file:" << abs_path;
cleanup(abs_path, create);
return nullptr;
}
// Align up to the nearest block size.
if (file_size % blocksz) {
file_size += blocksz - (file_size % blocksz);
}
if (create) {
if (!AllocateFile(file_fd, abs_path, blocksz, file_size, fs_type, std::move(progress))) {
LOG(ERROR) << "Failed to allocate file: " << abs_path << " of size: " << file_size
<< " bytes";
cleanup(abs_path, create);
return nullptr;
}
}
// f2fs may move the file blocks around.
if (!PinFile(file_fd, abs_path, fs_type)) {
cleanup(abs_path, create);
LOG(ERROR) << "Failed to pin the file in storage";
return nullptr;
}
// now allocate the FiemapWriter and start setting it up
FiemapUniquePtr fmap(new FiemapWriter());
switch (fs_type) {
case EXT4_SUPER_MAGIC:
case F2FS_SUPER_MAGIC:
if (!ReadFiemap(file_fd, abs_path, &fmap->extents_)) {
LOG(ERROR) << "Failed to read fiemap of file: " << abs_path;
cleanup(abs_path, create);
return nullptr;
}
break;
case MSDOS_SUPER_MAGIC:
if (!ReadFibmap(file_fd, abs_path, &fmap->extents_)) {
LOG(ERROR) << "Failed to read fibmap of file: " << abs_path;
cleanup(abs_path, create);
return nullptr;
}
break;
}
fmap->file_path_ = abs_path;
fmap->bdev_path_ = bdev_path;
fmap->file_size_ = file_size;
fmap->bdev_size_ = bdevsz;
fmap->fs_type_ = fs_type;
fmap->block_size_ = blocksz;
LOG(VERBOSE) << "Successfully created FiemapWriter for file " << abs_path << " on block device "
<< bdev_path;
return fmap;
}
} // namespace fiemap_writer
} // namespace android

541
fs_mgr/libfiemap_writer/fiemap_writer_test.cpp
View File

@ -1,541 +0,0 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fcntl.h>
#include <inttypes.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>
#include <string>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/stringprintf.h>
#include <android-base/unique_fd.h>
#include <gtest/gtest.h>
#include <libdm/loop_control.h>
#include <libfiemap_writer/fiemap_writer.h>
#include <libfiemap_writer/split_fiemap_writer.h>
#include "utility.h"
namespace android {
namespace fiemap_writer {
using namespace std;
using namespace std::string_literals;
using namespace android::fiemap_writer;
using unique_fd = android::base::unique_fd;
using LoopDevice = android::dm::LoopDevice;
std::string gTestDir;
uint64_t testfile_size = 536870912; // default of 512MiB
size_t gBlockSize = 0;
class FiemapWriterTest : public ::testing::Test {
protected:
void SetUp() override {
const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
testfile = gTestDir + "/"s + tinfo->name();
}
void TearDown() override { unlink(testfile.c_str()); }
// name of the file we use for testing
std::string testfile;
};
class SplitFiemapTest : public ::testing::Test {
protected:
void SetUp() override {
const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
testfile = gTestDir + "/"s + tinfo->name();
}
void TearDown() override {
std::string message;
if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) {
cerr << "Could not remove all split files: " << message;
}
}
// name of the file we use for testing
std::string testfile;
};
TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) {
// Try creating a file of size ~100TB but aligned to
// 512 byte to make sure block alignment tests don't
// fail.
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1099511627997184);
EXPECT_EQ(fptr, nullptr);
EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
EXPECT_EQ(errno, ENOENT);
}
TEST_F(FiemapWriterTest, CreateUnalignedFile) {
// Try creating a file of size 4097 bytes which is guaranteed
// to be unaligned to all known block sizes.
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize + 1);
ASSERT_NE(fptr, nullptr);
ASSERT_EQ(fptr->size(), gBlockSize * 2);
}
TEST_F(FiemapWriterTest, CheckFilePath) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
ASSERT_NE(fptr, nullptr);
EXPECT_EQ(fptr->size(), gBlockSize);
EXPECT_EQ(fptr->file_path(), testfile);
EXPECT_EQ(access(testfile.c_str(), F_OK), 0);
}
TEST_F(FiemapWriterTest, CheckFileSize) {
// Create a large-ish file and test that the expected size matches.
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1024 * 1024 * 16);
ASSERT_NE(fptr, nullptr);
struct stat s;
ASSERT_EQ(stat(testfile.c_str(), &s), 0);
EXPECT_EQ(static_cast<uint64_t>(s.st_size), fptr->size());
}
TEST_F(FiemapWriterTest, CheckProgress) {
std::vector<uint64_t> expected;
size_t invocations = 0;
auto callback = [&](uint64_t done, uint64_t total) -> bool {
if (invocations >= expected.size()) {
return false;
}
EXPECT_EQ(done, expected[invocations]);
EXPECT_EQ(total, gBlockSize);
invocations++;
return true;
};
expected.push_back(gBlockSize);
auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
EXPECT_NE(ptr, nullptr);
EXPECT_EQ(invocations, expected.size());
}
TEST_F(FiemapWriterTest, CheckPinning) {
auto ptr = FiemapWriter::Open(testfile, 4096);
ASSERT_NE(ptr, nullptr);
EXPECT_TRUE(FiemapWriter::HasPinnedExtents(testfile));
}
TEST_F(FiemapWriterTest, CheckBlockDevicePath) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
EXPECT_EQ(fptr->size(), gBlockSize);
EXPECT_EQ(fptr->bdev_path().find("/dev/block/"), size_t(0));
EXPECT_EQ(fptr->bdev_path().find("/dev/block/dm-"), string::npos);
}
TEST_F(FiemapWriterTest, CheckFileCreated) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 32768);
ASSERT_NE(fptr, nullptr);
unique_fd fd(open(testfile.c_str(), O_RDONLY));
EXPECT_GT(fd, -1);
}
TEST_F(FiemapWriterTest, CheckFileSizeActual) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
ASSERT_NE(fptr, nullptr);
struct stat sb;
ASSERT_EQ(stat(testfile.c_str(), &sb), 0);
EXPECT_GE(sb.st_size, testfile_size);
}
TEST_F(FiemapWriterTest, CheckFileExtents) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
ASSERT_NE(fptr, nullptr);
EXPECT_GT(fptr->extents().size(), 0);
}
TEST_F(FiemapWriterTest, ExistingFile) {
// Create the file.
{ ASSERT_NE(FiemapWriter::Open(testfile, gBlockSize), nullptr); }
// Test that we can still open it.
{
auto ptr = FiemapWriter::Open(testfile, 0, false);
ASSERT_NE(ptr, nullptr);
EXPECT_GT(ptr->extents().size(), 0);
}
}
TEST_F(FiemapWriterTest, FileDeletedOnError) {
auto callback = [](uint64_t, uint64_t) -> bool { return false; };
auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
EXPECT_EQ(ptr, nullptr);
EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
EXPECT_EQ(errno, ENOENT);
}
TEST_F(FiemapWriterTest, MaxBlockSize) {
ASSERT_GT(DetermineMaximumFileSize(testfile), 0);
}
TEST_F(FiemapWriterTest, FibmapBlockAddressing) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
ASSERT_NE(fptr, nullptr);
switch (fptr->fs_type()) {
case F2FS_SUPER_MAGIC:
case EXT4_SUPER_MAGIC:
// Skip the test for FIEMAP supported filesystems. This is really
// because f2fs/ext4 have caches that seem to defeat reading back
// directly from the block device, and writing directly is too
// dangerous.
std::cout << "Skipping test, filesystem does not use FIBMAP\n";
return;
}
bool uses_dm;
std::string bdev_path;
ASSERT_TRUE(FiemapWriter::GetBlockDeviceForFile(testfile, &bdev_path, &uses_dm));
if (uses_dm) {
// We could use a device-mapper wrapper here to bypass encryption, but
// really this test is for FIBMAP correctness on VFAT (where encryption
// is never used), so we don't bother.
std::cout << "Skipping test, block device is metadata encrypted\n";
return;
}
std::string data(fptr->size(), '\0');
for (size_t i = 0; i < data.size(); i++) {
data[i] = 'A' + static_cast<char>(data.size() % 26);
}
{
unique_fd fd(open(testfile.c_str(), O_WRONLY | O_CLOEXEC));
ASSERT_GE(fd, 0);
ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size()));
ASSERT_EQ(fsync(fd), 0);
}
ASSERT_FALSE(fptr->extents().empty());
const auto& first_extent = fptr->extents()[0];
unique_fd bdev(open(fptr->bdev_path().c_str(), O_RDONLY | O_CLOEXEC));
ASSERT_GE(bdev, 0);
off_t where = first_extent.fe_physical;
ASSERT_EQ(lseek(bdev, where, SEEK_SET), where);
// Note: this will fail on encrypted folders.
std::string actual(data.size(), '\0');
ASSERT_GE(first_extent.fe_length, data.size());
ASSERT_TRUE(android::base::ReadFully(bdev, actual.data(), actual.size()));
EXPECT_EQ(memcmp(actual.data(), data.data(), data.size()), 0);
}
TEST_F(SplitFiemapTest, Create) {
auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
ASSERT_NE(ptr, nullptr);
auto extents = ptr->extents();
// Destroy the fiemap, closing file handles. This should not delete them.
ptr = nullptr;
std::vector<std::string> files;
ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files));
for (const auto& path : files) {
EXPECT_EQ(access(path.c_str(), F_OK), 0);
}
ASSERT_GE(extents.size(), files.size());
}
TEST_F(SplitFiemapTest, Open) {
{
auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
ASSERT_NE(ptr, nullptr);
}
auto ptr = SplitFiemap::Open(testfile);
ASSERT_NE(ptr, nullptr);
auto extents = ptr->extents();
ASSERT_GE(extents.size(), 24);
}
TEST_F(SplitFiemapTest, DeleteOnFail) {
auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 100, 1);
ASSERT_EQ(ptr, nullptr);
std::string first_file = testfile + ".0001";
ASSERT_NE(access(first_file.c_str(), F_OK), 0);
ASSERT_EQ(errno, ENOENT);
ASSERT_NE(access(testfile.c_str(), F_OK), 0);
ASSERT_EQ(errno, ENOENT);
}
static string ReadSplitFiles(const std::string& base_path, size_t num_files) {
std::string result;
for (int i = 0; i < num_files; i++) {
std::string path = base_path + android::base::StringPrintf(".%04d", i);
std::string data;
if (!android::base::ReadFileToString(path, &data)) {
return {};
}
result += data;
}
return result;
}
TEST_F(SplitFiemapTest, WriteWholeFile) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
auto actual = ReadSplitFiles(testfile, 3);
ASSERT_EQ(expected.size(), actual.size());
EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}
TEST_F(SplitFiemapTest, WriteFileInChunks1) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
// Write in chunks of 1000 (so some writes straddle the boundary of two
// files).
size_t bytes_written = 0;
while (bytes_written < kSize) {
size_t to_write = std::min(kSize - bytes_written, (size_t)1000);
char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
ASSERT_TRUE(ptr->Write(data, to_write));
bytes_written += to_write;
}
std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
auto actual = ReadSplitFiles(testfile, 3);
ASSERT_EQ(expected.size(), actual.size());
EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}
TEST_F(SplitFiemapTest, WriteFileInChunks2) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
// Write in chunks of 32KiB so every write is exactly at the end of the
// current file.
size_t bytes_written = 0;
while (bytes_written < kSize) {
size_t to_write = std::min(kSize - bytes_written, kChunkSize);
char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
ASSERT_TRUE(ptr->Write(data, to_write));
bytes_written += to_write;
}
std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
auto actual = ReadSplitFiles(testfile, 3);
ASSERT_EQ(expected.size(), actual.size());
EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}
TEST_F(SplitFiemapTest, WritePastEnd) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
ASSERT_FALSE(ptr->Write(buffer.get(), kSize));
}
class VerifyBlockWritesExt4 : public ::testing::Test {
// 2GB Filesystem and 4k block size by default
static constexpr uint64_t block_size = 4096;
static constexpr uint64_t fs_size = 2147483648;
protected:
void SetUp() override {
fs_path = std::string(getenv("TMPDIR")) + "/ext4_2G.img";
uint64_t count = fs_size / block_size;
std::string dd_cmd =
::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
" count=%" PRIu64 " > /dev/null 2>&1",
fs_path.c_str(), block_size, count);
std::string mkfs_cmd =
::android::base::StringPrintf("/system/bin/mkfs.ext4 -q %s", fs_path.c_str());
// create mount point
mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt";
ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0);
// create file for the file system
int ret = system(dd_cmd.c_str());
ASSERT_EQ(ret, 0);
// Get and attach a loop device to the filesystem we created
LoopDevice loop_dev(fs_path);
ASSERT_TRUE(loop_dev.valid());
// create file system
ret = system(mkfs_cmd.c_str());
ASSERT_EQ(ret, 0);
// mount the file system
ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "ext4", 0, nullptr), 0);
}
void TearDown() override {
umount(mntpoint.c_str());
rmdir(mntpoint.c_str());
unlink(fs_path.c_str());
}
std::string mntpoint;
std::string fs_path;
};
class VerifyBlockWritesF2fs : public ::testing::Test {
// 2GB Filesystem and 4k block size by default
static constexpr uint64_t block_size = 4096;
static constexpr uint64_t fs_size = 2147483648;
protected:
void SetUp() override {
fs_path = std::string(getenv("TMPDIR")) + "/f2fs_2G.img";
uint64_t count = fs_size / block_size;
std::string dd_cmd =
::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
" count=%" PRIu64 " > /dev/null 2>&1",
fs_path.c_str(), block_size, count);
std::string mkfs_cmd =
::android::base::StringPrintf("/system/bin/make_f2fs -q %s", fs_path.c_str());
// create mount point
mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt";
ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0);
// create file for the file system
int ret = system(dd_cmd.c_str());
ASSERT_EQ(ret, 0);
// Get and attach a loop device to the filesystem we created
LoopDevice loop_dev(fs_path);
ASSERT_TRUE(loop_dev.valid());
// create file system
ret = system(mkfs_cmd.c_str());
ASSERT_EQ(ret, 0);
// mount the file system
ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "f2fs", 0, nullptr), 0);
}
void TearDown() override {
umount(mntpoint.c_str());
rmdir(mntpoint.c_str());
unlink(fs_path.c_str());
}
std::string mntpoint;
std::string fs_path;
};
bool DetermineBlockSize() {
struct statfs s;
if (statfs(gTestDir.c_str(), &s)) {
std::cerr << "Could not call statfs: " << strerror(errno) << "\n";
return false;
}
if (!s.f_bsize) {
std::cerr << "Invalid block size: " << s.f_bsize << "\n";
return false;
}
gBlockSize = s.f_bsize;
return true;
}
} // namespace fiemap_writer
} // namespace android
using namespace android::fiemap_writer;
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
if (argc > 1 && argv[1] == "-h"s) {
cerr << "Usage: [test_dir] [file_size]\n";
cerr << "\n";
cerr << "Note: test_dir must be a writable, unencrypted directory.\n";
exit(EXIT_FAILURE);
}
::android::base::InitLogging(argv, ::android::base::StderrLogger);
std::string root_dir = "/data/local/unencrypted";
if (access(root_dir.c_str(), F_OK)) {
root_dir = "/data";
}
std::string tempdir = root_dir + "/XXXXXX"s;
if (!mkdtemp(tempdir.data())) {
cerr << "unable to create tempdir on " << root_dir << "\n";
exit(EXIT_FAILURE);
}
if (!android::base::Realpath(tempdir, &gTestDir)) {
cerr << "unable to find realpath for " << tempdir;
exit(EXIT_FAILURE);
}
if (argc > 2) {
testfile_size = strtoull(argv[2], NULL, 0);
if (testfile_size == ULLONG_MAX) {
testfile_size = 512 * 1024 * 1024;
}
}
if (!DetermineBlockSize()) {
exit(EXIT_FAILURE);
}
auto result = RUN_ALL_TESTS();
std::string cmd = "rm -rf " + gTestDir;
system(cmd.c_str());
return result;
}

118
fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h
View File

@ -1,118 +0,0 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <linux/fiemap.h>
#include <stdint.h>
#include <sys/types.h>
#include <unistd.h>
#include <functional>
#include <string>
#include <vector>
#include <android-base/unique_fd.h>
namespace android {
namespace fiemap_writer {
class FiemapWriter;
using FiemapUniquePtr = std::unique_ptr<FiemapWriter>;
class FiemapWriter final {
public:
// Factory method for FiemapWriter.
// The method returns FiemapUniquePtr that contains all the data necessary to be able to write
// to the given file directly using raw block i/o. The optional progress callback will be
// invoked, if create is true, while the file is being initialized. It receives the bytes
// written and the number of total bytes. If the callback returns false, the operation will
// fail.
//
// Note: when create is true, the file size will be aligned up to the nearest file system
// block.
static FiemapUniquePtr Open(const std::string& file_path, uint64_t file_size,
bool create = true,
std::function<bool(uint64_t, uint64_t)> progress = {});
// Check that a file still has the same extents since it was last opened with FiemapWriter,
// assuming the file was not resized outside of FiemapWriter. Returns false either on error
// or if the file was not pinned.
//
// This will always return true on Ext4. On F2FS, it will return true if either of the
// following cases are true:
// - The file was never pinned.
// - The file is pinned and has not been moved by the GC.
// Thus, this method should only be called for pinned files (such as those returned by
// FiemapWriter::Open).
static bool HasPinnedExtents(const std::string& file_path);
// Returns the underlying block device of a file. This will look past device-mapper layers
// as long as each layer would not change block mappings (i.e., dm-crypt, dm-bow, and dm-
// default-key tables are okay; dm-linear is not). If a mapping such as dm-linear is found,
// it will be returned in place of any physical block device.
//
// It is the caller's responsibility to check whether the returned block device is acceptable.
// Gsid, for example, will only accept /dev/block/by-name/userdata as the bottom device.
// Callers can check the device name (dm- or loop prefix), inspect sysfs, or compare the major
// number against a boot device.
//
// If device-mapper nodes were encountered, then |uses_dm| will be set to true.
static bool GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path,
bool* uses_dm = nullptr);
~FiemapWriter() = default;
const std::string& file_path() const { return file_path_; };
uint64_t size() const { return file_size_; };
const std::string& bdev_path() const { return bdev_path_; };
uint64_t block_size() const { return block_size_; };
const std::vector<struct fiemap_extent>& extents() { return extents_; };
uint32_t fs_type() const { return fs_type_; }
// Non-copyable & Non-movable
FiemapWriter(const FiemapWriter&) = delete;
FiemapWriter& operator=(const FiemapWriter&) = delete;
FiemapWriter& operator=(FiemapWriter&&) = delete;
FiemapWriter(FiemapWriter&&) = delete;
private:
// Name of the file managed by this class.
std::string file_path_;
// Block device on which we have created the file.
std::string bdev_path_;
// Size in bytes of the file this class is writing
uint64_t file_size_;
// total size in bytes of the block device
uint64_t bdev_size_;
// Filesystem type where the file is being created.
// See: <uapi/linux/magic.h> for filesystem magic numbers
uint32_t fs_type_;
// block size as reported by the kernel of the underlying block device;
uint64_t block_size_;
// This file's fiemap
std::vector<struct fiemap_extent> extents_;
FiemapWriter() = default;
};
} // namespace fiemap_writer
} // namespace android

97
fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h
View File

@ -1,97 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <stdint.h>
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include <android-base/unique_fd.h>
#include "fiemap_writer.h"
namespace android {
namespace fiemap_writer {
// Wrapper around FiemapWriter that is able to split images across files if
// necessary.
class SplitFiemap final {
public:
using ProgressCallback = std::function<bool(uint64_t, uint64_t)>;
// Create a new split fiemap file. If |max_piece_size| is 0, the number of
// pieces will be determined automatically by detecting the filesystem.
// Otherwise, the file will be split evenly (with the remainder in the
// final file).
static std::unique_ptr<SplitFiemap> Create(const std::string& file_path, uint64_t file_size,
uint64_t max_piece_size,
ProgressCallback progress = {});
// Open an existing split fiemap file.
static std::unique_ptr<SplitFiemap> Open(const std::string& file_path);
~SplitFiemap();
// Return a list of all files created for a split file.
static bool GetSplitFileList(const std::string& file_path, std::vector<std::string>* list);
// Destroy all components of a split file. If the root file does not exist,
// this returns true and does not report an error.
static bool RemoveSplitFiles(const std::string& file_path, std::string* message = nullptr);
// Return whether all components of a split file still have pinned extents.
bool HasPinnedExtents() const;
// Helper method for writing data that spans files. Note there is no seek
// method (yet); this starts at 0 and increments the position by |bytes|.
bool Write(const void* data, uint64_t bytes);
// Flush all writes to all split files.
bool Flush();
const std::vector<struct fiemap_extent>& extents();
uint32_t block_size() const;
uint64_t size() const { return total_size_; }
const std::string& bdev_path() const;
// Non-copyable & Non-movable
SplitFiemap(const SplitFiemap&) = delete;
SplitFiemap& operator=(const SplitFiemap&) = delete;
SplitFiemap& operator=(SplitFiemap&&) = delete;
SplitFiemap(SplitFiemap&&) = delete;
private:
SplitFiemap() = default;
void AddFile(FiemapUniquePtr&& file);
bool creating_ = false;
std::string list_file_;
std::vector<FiemapUniquePtr> files_;
std::vector<struct fiemap_extent> extents_;
uint64_t total_size_ = 0;
// Most recently open file and position for Write().
size_t cursor_index_ = 0;
uint64_t cursor_file_pos_ = 0;
android::base::unique_fd cursor_fd_;
};
} // namespace fiemap_writer
} // namespace android

298
fs_mgr/libfiemap_writer/split_fiemap_writer.cpp
View File

@ -1,298 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <libfiemap_writer/split_fiemap_writer.h>
#include <fcntl.h>
#include <stdint.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <memory>
#include <string>
#include <vector>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include "utility.h"
namespace android {
namespace fiemap_writer {
using android::base::unique_fd;
// We use a four-digit suffix at the end of filenames.
static const size_t kMaxFilePieces = 500;
std::unique_ptr<SplitFiemap> SplitFiemap::Create(const std::string& file_path, uint64_t file_size,
uint64_t max_piece_size,
ProgressCallback progress) {
if (!file_size) {
LOG(ERROR) << "Cannot create a fiemap for a 0-length file: " << file_path;
return nullptr;
}
if (!max_piece_size) {
max_piece_size = DetermineMaximumFileSize(file_path);
if (!max_piece_size) {
LOG(ERROR) << "Could not determine maximum file size for " << file_path;
return nullptr;
}
}
// Call |progress| only when the total percentage would significantly change.
int permille = -1;
uint64_t total_bytes_written = 0;
auto on_progress = [&](uint64_t written, uint64_t) -> bool {
uint64_t actual_written = total_bytes_written + written;
int new_permille = (actual_written * 1000) / file_size;
if (new_permille != permille && actual_written < file_size) {
if (progress && !progress(actual_written, file_size)) {
return false;
}
permille = new_permille;
}
return true;
};
std::unique_ptr<SplitFiemap> out(new SplitFiemap());
out->creating_ = true;
out->list_file_ = file_path;
// Create the split files.
uint64_t remaining_bytes = file_size;
while (remaining_bytes) {
if (out->files_.size() >= kMaxFilePieces) {
LOG(ERROR) << "Requested size " << file_size << " created too many split files";
return nullptr;
}
std::string chunk_path =
android::base::StringPrintf("%s.%04d", file_path.c_str(), (int)out->files_.size());
uint64_t chunk_size = std::min(max_piece_size, remaining_bytes);
auto writer = FiemapWriter::Open(chunk_path, chunk_size, true, on_progress);
if (!writer) {
return nullptr;
}
// To make sure the alignment doesn't create too much inconsistency, we
// account the *actual* size, not the requested size.
total_bytes_written += writer->size();
// writer->size() is block size aligned and could be bigger than remaining_bytes
// If remaining_bytes is bigger, set remaining_bytes to 0 to avoid underflow error.
remaining_bytes = remaining_bytes > writer->size() ? (remaining_bytes - writer->size()) : 0;
out->AddFile(std::move(writer));
}
// Create the split file list.
unique_fd fd(open(out->list_file_.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0660));
if (fd < 0) {
PLOG(ERROR) << "Failed to open " << file_path;
return nullptr;
}
for (const auto& writer : out->files_) {
std::string line = android::base::Basename(writer->file_path()) + "\n";
if (!android::base::WriteFully(fd, line.data(), line.size())) {
PLOG(ERROR) << "Write failed " << file_path;
return nullptr;
}
}
// Unset this bit, so we don't unlink on destruction.
out->creating_ = false;
return out;
}
std::unique_ptr<SplitFiemap> SplitFiemap::Open(const std::string& file_path) {
std::vector<std::string> files;
if (!GetSplitFileList(file_path, &files)) {
return nullptr;
}
std::unique_ptr<SplitFiemap> out(new SplitFiemap());
out->list_file_ = file_path;
for (const auto& file : files) {
auto writer = FiemapWriter::Open(file, 0, false);
if (!writer) {
// Error was logged in Open().
return nullptr;
}
out->AddFile(std::move(writer));
}
return out;
}
bool SplitFiemap::GetSplitFileList(const std::string& file_path, std::vector<std::string>* list) {
// This is not the most efficient thing, but it is simple and recovering
// the fiemap/fibmap is much more expensive.
std::string contents;
if (!android::base::ReadFileToString(file_path, &contents, true)) {
PLOG(ERROR) << "Error reading file: " << file_path;
return false;
}
std::vector<std::string> names = android::base::Split(contents, "\n");
std::string dir = android::base::Dirname(file_path);
for (const auto& name : names) {
if (!name.empty()) {
list->emplace_back(dir + "/" + name);
}
}
return true;
}
bool SplitFiemap::RemoveSplitFiles(const std::string& file_path, std::string* message) {
// Early exit if this does not exist, and do not report an error.
if (access(file_path.c_str(), F_OK) && errno == ENOENT) {
return true;
}
bool ok = true;
std::vector<std::string> files;
if (GetSplitFileList(file_path, &files)) {
for (const auto& file : files) {
ok &= android::base::RemoveFileIfExists(file, message);
}
}
ok &= android::base::RemoveFileIfExists(file_path, message);
return ok;
}
bool SplitFiemap::HasPinnedExtents() const {
for (const auto& file : files_) {
if (!FiemapWriter::HasPinnedExtents(file->file_path())) {
return false;
}
}
return true;
}
const std::vector<struct fiemap_extent>& SplitFiemap::extents() {
if (extents_.empty()) {
for (const auto& file : files_) {
const auto& extents = file->extents();
extents_.insert(extents_.end(), extents.begin(), extents.end());
}
}
return extents_;
}
bool SplitFiemap::Write(const void* data, uint64_t bytes) {
// Open the current file.
FiemapWriter* file = files_[cursor_index_].get();
const uint8_t* data_ptr = reinterpret_cast<const uint8_t*>(data);
uint64_t bytes_remaining = bytes;
while (bytes_remaining) {
// How many bytes can we write into the current file?
uint64_t file_bytes_left = file->size() - cursor_file_pos_;
if (!file_bytes_left) {
if (cursor_index_ == files_.size() - 1) {
LOG(ERROR) << "write past end of file requested";
return false;
}
// No space left in the current file, but we have more files to
// use, so prep the next one.
cursor_fd_ = {};
cursor_file_pos_ = 0;
file = files_[++cursor_index_].get();
file_bytes_left = file->size();
}
// Open the current file if it's not open.
if (cursor_fd_ < 0) {
cursor_fd_.reset(open(file->file_path().c_str(), O_CLOEXEC | O_WRONLY));
if (cursor_fd_ < 0) {
PLOG(ERROR) << "open failed: " << file->file_path();
return false;
}
CHECK(cursor_file_pos_ == 0);
}
if (!FiemapWriter::HasPinnedExtents(file->file_path())) {
LOG(ERROR) << "file is no longer pinned: " << file->file_path();
return false;
}
uint64_t bytes_to_write = std::min(file_bytes_left, bytes_remaining);
if (!android::base::WriteFully(cursor_fd_, data_ptr, bytes_to_write)) {
PLOG(ERROR) << "write failed: " << file->file_path();
return false;
}
data_ptr += bytes_to_write;
bytes_remaining -= bytes_to_write;
cursor_file_pos_ += bytes_to_write;
}
// If we've reached the end of the current file, close it for sanity.
if (cursor_file_pos_ == file->size()) {
cursor_fd_ = {};
}
return true;
}
bool SplitFiemap::Flush() {
for (const auto& file : files_) {
unique_fd fd(open(file->file_path().c_str(), O_RDONLY | O_CLOEXEC));
if (fd < 0) {
PLOG(ERROR) << "open failed: " << file->file_path();
return false;
}
if (fsync(fd)) {
PLOG(ERROR) << "fsync failed: " << file->file_path();
return false;
}
}
return true;
}
SplitFiemap::~SplitFiemap() {
if (!creating_) {
return;
}
// We failed to finish creating, so unlink everything.
unlink(list_file_.c_str());
for (auto&& file : files_) {
std::string path = file->file_path();
file = nullptr;
unlink(path.c_str());
}
}
void SplitFiemap::AddFile(FiemapUniquePtr&& file) {
total_size_ += file->size();
files_.emplace_back(std::move(file));
}
uint32_t SplitFiemap::block_size() const {
return files_[0]->block_size();
}
const std::string& SplitFiemap::bdev_path() const {
return files_[0]->bdev_path();
}
} // namespace fiemap_writer
} // namespace android

BIN
fs_mgr/libfiemap_writer/testdata/file_32k vendored
View File

Binary file not shown.

BIN
fs_mgr/libfiemap_writer/testdata/file_4k vendored
View File

Binary file not shown.

BIN
fs_mgr/libfiemap_writer/testdata/unaligned_file vendored
View File

Binary file not shown.

66
fs_mgr/libfiemap_writer/utility.cpp
View File

@ -1,66 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "utility.h"
#include <stdint.h>
#include <sys/vfs.h>
#include <unistd.h>
#include <android-base/logging.h>
#include <libfiemap_writer/fiemap_writer.h>
namespace android {
namespace fiemap_writer {
uint64_t DetermineMaximumFileSize(const std::string& file_path) {
// Create the smallest file possible (one block).
auto writer = FiemapWriter::Open(file_path, 1);
if (!writer) {
return 0;
}
uint64_t result = 0;
switch (writer->fs_type()) {
case EXT4_SUPER_MAGIC:
// The minimum is 16GiB, so just report that. If we wanted we could parse the
// superblock and figure out if 64-bit support is enabled.
result = 17179869184ULL;
break;
case F2FS_SUPER_MAGIC:
// Formula is from https://www.kernel.org/doc/Documentation/filesystems/f2fs.txt
// 4KB * (923 + 2 * 1018 + 2 * 1018 * 1018 + 1018 * 1018 * 1018) := 3.94TB.
result = 4329690886144ULL;
break;
case MSDOS_SUPER_MAGIC:
// 4GB-1, which we want aligned to the block size.
result = 4294967295;
result -= (result % writer->block_size());
break;
default:
LOG(ERROR) << "Unknown file system type: " << writer->fs_type();
break;
}
// Close and delete the temporary file.
writer = nullptr;
unlink(file_path.c_str());
return result;
}
} // namespace fiemap_writer
} // namespace android

30
fs_mgr/libfiemap_writer/utility.h
View File

@ -1,30 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <string>
namespace android {
namespace fiemap_writer {
// Given a file that will be created, determine the maximum size its containing
// filesystem allows. Note this is a theoretical maximum size; free space is
// ignored entirely.
uint64_t DetermineMaximumFileSize(const std::string& file_path);
} // namespace fiemap_writer
} // namespace android