mirror of https://gitee.com/openkylin/linux.git
f2fs: cache global IPU bio
In commit 8648de2c58 ("f2fs: add bio cache for IPU"), we added
f2fs_submit_ipu_bio() in __write_data_page() as below:
__write_data_page()
if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode)) {
f2fs_submit_ipu_bio(sbi, bio, page);
....
}
in order to avoid below deadlock:
Thread A Thread B
- __write_data_page (inode x, page y)
- f2fs_do_write_data_page
- set_page_writeback ---- set writeback flag in page y
- f2fs_inplace_write_data
- f2fs_balance_fs
- lock gc_mutex
- lock gc_mutex
- f2fs_gc
- do_garbage_collect
- gc_data_segment
- move_data_page
- f2fs_wait_on_page_writeback
- wait_on_page_writeback --- wait writeback of page y
However, the bio submission breaks the merge of IPU IOs.
So in this patch let's add a global bio cache for merged IPU pages,
then f2fs_wait_on_page_writeback() is able to submit bio if a
writebacked page is cached in global bio cache.
Signed-off-by: Chao Yu <yuchao0@huawei.com>
Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
This commit is contained in:
Chao Yu
Jaegeuk Kim
parent
688078e7f3
commit
0b20fcec86
179
fs/f2fs/data.c
179
fs/f2fs/data.c
|
|
@ -29,6 +29,7 @@
|
|||
#define NUM_PREALLOC_POST_READ_CTXS 128
|
||||
|
||||
static struct kmem_cache *bio_post_read_ctx_cache;
|
||||
static struct kmem_cache *bio_entry_slab;
|
||||
static mempool_t *bio_post_read_ctx_pool;
|
||||
|
||||
static bool __is_cp_guaranteed(struct page *page)
|
||||
|
|
@ -543,6 +544,126 @@ static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
|
|||
return io_type_is_mergeable(io, fio);
|
||||
}
|
||||
|
||||
static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio,
|
||||
struct page *page, enum temp_type temp)
|
||||
{
|
||||
struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
|
||||
struct bio_entry *be;
|
||||
|
||||
be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS);
|
||||
be->bio = bio;
|
||||
bio_get(bio);
|
||||
|
||||
if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE)
|
||||
f2fs_bug_on(sbi, 1);
|
||||
|
||||
down_write(&io->bio_list_lock);
|
||||
list_add_tail(&be->list, &io->bio_list);
|
||||
up_write(&io->bio_list_lock);
|
||||
}
|
||||
|
||||
static void del_bio_entry(struct bio_entry *be)
|
||||
{
|
||||
list_del(&be->list);
|
||||
kmem_cache_free(bio_entry_slab, be);
|
||||
}
|
||||
|
||||
static int add_ipu_page(struct f2fs_sb_info *sbi, struct bio **bio,
|
||||
struct page *page)
|
||||
{
|
||||
enum temp_type temp;
|
||||
bool found = false;
|
||||
int ret = -EAGAIN;
|
||||
|
||||
for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
|
||||
struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
|
||||
struct list_head *head = &io->bio_list;
|
||||
struct bio_entry *be;
|
||||
|
||||
down_write(&io->bio_list_lock);
|
||||
list_for_each_entry(be, head, list) {
|
||||
if (be->bio != *bio)
|
||||
continue;
|
||||
|
||||
found = true;
|
||||
|
||||
if (bio_add_page(*bio, page, PAGE_SIZE, 0) == PAGE_SIZE) {
|
||||
ret = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
/* bio is full */
|
||||
del_bio_entry(be);
|
||||
__submit_bio(sbi, *bio, DATA);
|
||||
break;
|
||||
}
|
||||
up_write(&io->bio_list_lock);
|
||||
}
|
||||
|
||||
if (ret) {
|
||||
bio_put(*bio);
|
||||
*bio = NULL;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
|
||||
struct bio **bio, struct page *page)
|
||||
{
|
||||
enum temp_type temp;
|
||||
bool found = false;
|
||||
struct bio *target = bio ? *bio : NULL;
|
||||
|
||||
for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
|
||||
struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
|
||||
struct list_head *head = &io->bio_list;
|
||||
struct bio_entry *be;
|
||||
|
||||
if (list_empty(head))
|
||||
continue;
|
||||
|
||||
down_read(&io->bio_list_lock);
|
||||
list_for_each_entry(be, head, list) {
|
||||
if (target)
|
||||
found = (target == be->bio);
|
||||
else
|
||||
found = __has_merged_page(be->bio, NULL,
|
||||
page, 0);
|
||||
if (found)
|
||||
break;
|
||||
}
|
||||
up_read(&io->bio_list_lock);
|
||||
|
||||
if (!found)
|
||||
continue;
|
||||
|
||||
found = false;
|
||||
|
||||
down_write(&io->bio_list_lock);
|
||||
list_for_each_entry(be, head, list) {
|
||||
if (target)
|
||||
found = (target == be->bio);
|
||||
else
|
||||
found = __has_merged_page(be->bio, NULL,
|
||||
page, 0);
|
||||
if (found) {
|
||||
target = be->bio;
|
||||
del_bio_entry(be);
|
||||
break;
|
||||
}
|
||||
}
|
||||
up_write(&io->bio_list_lock);
|
||||
}
|
||||
|
||||
if (found)
|
||||
__submit_bio(sbi, target, DATA);
|
||||
if (bio && *bio) {
|
||||
bio_put(*bio);
|
||||
*bio = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
int f2fs_merge_page_bio(struct f2fs_io_info *fio)
|
||||
{
|
||||
struct bio *bio = *fio->bio;
|
||||
|
|
@ -557,20 +678,17 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio)
|
|||
f2fs_trace_ios(fio, 0);
|
||||
|
||||
if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block,
|
||||
fio->new_blkaddr)) {
|
||||
__submit_bio(fio->sbi, bio, fio->type);
|
||||
bio = NULL;
|
||||
}
|
||||
fio->new_blkaddr))
|
||||
f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL);
|
||||
alloc_new:
|
||||
if (!bio) {
|
||||
bio = __bio_alloc(fio, BIO_MAX_PAGES);
|
||||
bio_set_op_attrs(bio, fio->op, fio->op_flags);
|
||||
}
|
||||
|
||||
if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
|
||||
__submit_bio(fio->sbi, bio, fio->type);
|
||||
bio = NULL;
|
||||
goto alloc_new;
|
||||
add_bio_entry(fio->sbi, bio, page, fio->temp);
|
||||
} else {
|
||||
if (add_ipu_page(fio->sbi, &bio, page))
|
||||
goto alloc_new;
|
||||
}
|
||||
|
||||
if (fio->io_wbc)
|
||||
|
|
@ -584,19 +702,6 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void f2fs_submit_ipu_bio(struct f2fs_sb_info *sbi, struct bio **bio,
|
||||
struct page *page)
|
||||
{
|
||||
if (!bio)
|
||||
return;
|
||||
|
||||
if (!__has_merged_page(*bio, NULL, page, 0))
|
||||
return;
|
||||
|
||||
__submit_bio(sbi, *bio, DATA);
|
||||
*bio = NULL;
|
||||
}
|
||||
|
||||
void f2fs_submit_page_write(struct f2fs_io_info *fio)
|
||||
{
|
||||
struct f2fs_sb_info *sbi = fio->sbi;
|
||||
|
|
@ -2215,14 +2320,12 @@ static int __write_data_page(struct page *page, bool *submitted,
|
|||
|
||||
unlock_page(page);
|
||||
if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
|
||||
!F2FS_I(inode)->cp_task) {
|
||||
f2fs_submit_ipu_bio(sbi, bio, page);
|
||||
!F2FS_I(inode)->cp_task)
|
||||
f2fs_balance_fs(sbi, need_balance_fs);
|
||||
}
|
||||
|
||||
if (unlikely(f2fs_cp_error(sbi))) {
|
||||
f2fs_submit_ipu_bio(sbi, bio, page);
|
||||
f2fs_submit_merged_write(sbi, DATA);
|
||||
f2fs_submit_merged_ipu_write(sbi, bio, NULL);
|
||||
submitted = NULL;
|
||||
}
|
||||
|
||||
|
|
@ -2342,13 +2445,11 @@ static int f2fs_write_cache_pages(struct address_space *mapping,
|
|||
}
|
||||
|
||||
if (PageWriteback(page)) {
|
||||
if (wbc->sync_mode != WB_SYNC_NONE) {
|
||||
if (wbc->sync_mode != WB_SYNC_NONE)
|
||||
f2fs_wait_on_page_writeback(page,
|
||||
DATA, true, true);
|
||||
f2fs_submit_ipu_bio(sbi, &bio, page);
|
||||
} else {
|
||||
else
|
||||
goto continue_unlock;
|
||||
}
|
||||
}
|
||||
|
||||
if (!clear_page_dirty_for_io(page))
|
||||
|
|
@ -2406,7 +2507,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping,
|
|||
NULL, 0, DATA);
|
||||
/* submit cached bio of IPU write */
|
||||
if (bio)
|
||||
__submit_bio(sbi, bio, DATA);
|
||||
f2fs_submit_merged_ipu_write(sbi, &bio, NULL);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -3211,8 +3312,22 @@ int __init f2fs_init_post_read_processing(void)
|
|||
return -ENOMEM;
|
||||
}
|
||||
|
||||
void __exit f2fs_destroy_post_read_processing(void)
|
||||
void f2fs_destroy_post_read_processing(void)
|
||||
{
|
||||
mempool_destroy(bio_post_read_ctx_pool);
|
||||
kmem_cache_destroy(bio_post_read_ctx_cache);
|
||||
}
|
||||
|
||||
int __init f2fs_init_bio_entry_cache(void)
|
||||
{
|
||||
bio_entry_slab = f2fs_kmem_cache_create("bio_entry_slab",
|
||||
sizeof(struct bio_entry));
|
||||
if (!bio_entry_slab)
|
||||
return -ENOMEM;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void __exit f2fs_destroy_bio_entry_cache(void)
|
||||
{
|
||||
kmem_cache_destroy(bio_entry_slab);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1068,6 +1068,11 @@ struct f2fs_io_info {
|
|||
unsigned char version; /* version of the node */
|
||||
};
|
||||
|
||||
struct bio_entry {
|
||||
struct bio *bio;
|
||||
struct list_head list;
|
||||
};
|
||||
|
||||
#define is_read_io(rw) ((rw) == READ)
|
||||
struct f2fs_bio_info {
|
||||
struct f2fs_sb_info *sbi; /* f2fs superblock */
|
||||
|
|
@ -1077,6 +1082,8 @@ struct f2fs_bio_info {
|
|||
struct rw_semaphore io_rwsem; /* blocking op for bio */
|
||||
spinlock_t io_lock; /* serialize DATA/NODE IOs */
|
||||
struct list_head io_list; /* track fios */
|
||||
struct list_head bio_list; /* bio entry list head */
|
||||
struct rw_semaphore bio_list_lock; /* lock to protect bio entry list */
|
||||
};
|
||||
|
||||
#define FDEV(i) (sbi->devs[i])
|
||||
|
|
@ -3195,10 +3202,14 @@ void f2fs_destroy_checkpoint_caches(void);
|
|||
*/
|
||||
int f2fs_init_post_read_processing(void);
|
||||
void f2fs_destroy_post_read_processing(void);
|
||||
int f2fs_init_bio_entry_cache(void);
|
||||
void f2fs_destroy_bio_entry_cache(void);
|
||||
void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);
|
||||
void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
|
||||
struct inode *inode, struct page *page,
|
||||
nid_t ino, enum page_type type);
|
||||
void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
|
||||
struct bio **bio, struct page *page);
|
||||
void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi);
|
||||
int f2fs_submit_page_bio(struct f2fs_io_info *fio);
|
||||
int f2fs_merge_page_bio(struct f2fs_io_info *fio);
|
||||
|
|
|
|||
|
|
@ -3379,7 +3379,10 @@ void f2fs_wait_on_page_writeback(struct page *page,
|
|||
if (PageWriteback(page)) {
|
||||
struct f2fs_sb_info *sbi = F2FS_P_SB(page);
|
||||
|
||||
/* submit cached LFS IO */
|
||||
f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
|
||||
/* sbumit cached IPU IO */
|
||||
f2fs_submit_merged_ipu_write(sbi, NULL, page);
|
||||
if (ordered) {
|
||||
wait_on_page_writeback(page);
|
||||
f2fs_bug_on(sbi, locked && PageWriteback(page));
|
||||
|
|
|
|||
|
|
@ -3342,6 +3342,8 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
|
|||
sbi->write_io[i][j].bio = NULL;
|
||||
spin_lock_init(&sbi->write_io[i][j].io_lock);
|
||||
INIT_LIST_HEAD(&sbi->write_io[i][j].io_list);
|
||||
INIT_LIST_HEAD(&sbi->write_io[i][j].bio_list);
|
||||
init_rwsem(&sbi->write_io[i][j].bio_list_lock);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3753,8 +3755,13 @@ static int __init init_f2fs_fs(void)
|
|||
err = f2fs_init_post_read_processing();
|
||||
if (err)
|
||||
goto free_root_stats;
|
||||
err = f2fs_init_bio_entry_cache();
|
||||
if (err)
|
||||
goto free_post_read;
|
||||
return 0;
|
||||
|
||||
free_post_read:
|
||||
f2fs_destroy_post_read_processing();
|
||||
free_root_stats:
|
||||
f2fs_destroy_root_stats();
|
||||
unregister_filesystem(&f2fs_fs_type);
|
||||
|
|
@ -3778,6 +3785,7 @@ static int __init init_f2fs_fs(void)
|
|||
|
||||
static void __exit exit_f2fs_fs(void)
|
||||
{
|
||||
f2fs_destroy_bio_entry_cache();
|
||||
f2fs_destroy_post_read_processing();
|
||||
f2fs_destroy_root_stats();
|
||||
unregister_filesystem(&f2fs_fs_type);
|
||||
|
|
|
|||
Loading…
Reference in New Issue