p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvmet: use IOCB_NOWAIT for file-ns buffered I/O 

This patch optimizes read command behavior when file-ns configured
with buffered I/O. Instead of offloading the buffered I/O read operations
to the worker threads, we first issue the read operation with IOCB_NOWAIT
and try and access the data from the cache. Here we only offload the
request to the worker thread and complete the request in the worker
thread context when IOCB_NOWAIT request fails.

Signed-off-by: Chaitanya Kulkarni <chaitanya.kulkarni@wdc.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Chaitanya Kulkarni 2018-11-14 01:12:19 -05:00 committed by Jens Axboe
parent c09305ae49
commit 50a909db36
1 changed files with 84 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
drivers/nvme/target/io-cmd-file.c
View File

@ -83,17 +83,16 @@ static void nvmet_file_init_bvec(struct bio_vec *bv, struct sg_page_iter *iter)
}
static ssize_t nvmet_file_submit_bvec(struct nvmet_req *req, loff_t pos,
unsigned long nr_segs, size_t count)
unsigned long nr_segs, size_t count, int ki_flags)
{
struct kiocb *iocb = &req->f.iocb;
ssize_t (*call_iter)(struct kiocb *iocb, struct iov_iter *iter);
struct iov_iter iter;
int ki_flags = 0, rw;
ssize_t ret;
int rw;
if (req->cmd->rw.opcode == nvme_cmd_write) {
if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
ki_flags = IOCB_DSYNC;
ki_flags |= IOCB_DSYNC;
call_iter = req->ns->file->f_op->write_iter;
rw = WRITE;
} else {
@ -107,12 +106,7 @@ static ssize_t nvmet_file_submit_bvec(struct nvmet_req *req, loff_t pos,
iocb->ki_filp = req->ns->file;
iocb->ki_flags = ki_flags | iocb_flags(req->ns->file);
ret = call_iter(iocb, &iter);
if (ret != -EIOCBQUEUED && iocb->ki_complete)
iocb->ki_complete(iocb, ret, 0);
return ret;
return call_iter(iocb, &iter);
}
static void nvmet_file_io_done(struct kiocb *iocb, long ret, long ret2)
@ -130,7 +124,7 @@ static void nvmet_file_io_done(struct kiocb *iocb, long ret, long ret2)
NVME_SC_INTERNAL | NVME_SC_DNR : 0);
}
static void nvmet_file_execute_rw(struct nvmet_req *req)
static bool nvmet_file_execute_io(struct nvmet_req *req, int ki_flags)
{
ssize_t nr_bvec = DIV_ROUND_UP(req->data_len, PAGE_SIZE);
struct sg_page_iter sg_pg_iter;
@ -140,30 +134,14 @@ static void nvmet_file_execute_rw(struct nvmet_req *req)
ssize_t ret = 0;
loff_t pos;
if (!req->sg_cnt || !nr_bvec) {
nvmet_req_complete(req, 0);
return;
}
if (req->f.mpool_alloc && nr_bvec > NVMET_MAX_MPOOL_BVEC)
is_sync = true;
pos = le64_to_cpu(req->cmd->rw.slba) << req->ns->blksize_shift;
if (unlikely(pos + req->data_len > req->ns->size)) {
nvmet_req_complete(req, NVME_SC_LBA_RANGE | NVME_SC_DNR);
return;
}
if (nr_bvec > NVMET_MAX_INLINE_BIOVEC)
req->f.bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec),
GFP_KERNEL);
else
req->f.bvec = req->inline_bvec;
req->f.mpool_alloc = false;
if (unlikely(!req->f.bvec)) {
/* fallback under memory pressure */
req->f.bvec = mempool_alloc(req->ns->bvec_pool, GFP_KERNEL);
req->f.mpool_alloc = true;
if (nr_bvec > NVMET_MAX_MPOOL_BVEC)
is_sync = true;
return true;
}
memset(&req->f.iocb, 0, sizeof(struct kiocb));
@ -177,9 +155,10 @@ static void nvmet_file_execute_rw(struct nvmet_req *req)
if (unlikely(is_sync) &&
(nr_bvec - 1 == 0 || bv_cnt == NVMET_MAX_MPOOL_BVEC)) {
ret = nvmet_file_submit_bvec(req, pos, bv_cnt, len);
ret = nvmet_file_submit_bvec(req, pos, bv_cnt, len, 0);
if (ret < 0)
goto out;
goto complete;
pos += len;
bv_cnt = 0;
len = 0;
@ -187,30 +166,92 @@ static void nvmet_file_execute_rw(struct nvmet_req *req)
nr_bvec--;
}
if (WARN_ON_ONCE(total_len != req->data_len))
if (WARN_ON_ONCE(total_len != req->data_len)) {
ret = -EIO;
out:
if (unlikely(is_sync || ret)) {
nvmet_file_io_done(&req->f.iocb, ret < 0 ? ret : total_len, 0);
return;
goto complete;
}
req->f.iocb.ki_complete = nvmet_file_io_done;
nvmet_file_submit_bvec(req, pos, bv_cnt, total_len);
if (unlikely(is_sync)) {
ret = total_len;
goto complete;
}
/*
* A NULL ki_complete ask for synchronous execution, which we want
* for the IOCB_NOWAIT case.
*/
if (!(ki_flags & IOCB_NOWAIT))
req->f.iocb.ki_complete = nvmet_file_io_done;
ret = nvmet_file_submit_bvec(req, pos, bv_cnt, total_len, ki_flags);
switch (ret) {
case -EIOCBQUEUED:
return true;
case -EAGAIN:
if (WARN_ON_ONCE(!(ki_flags & IOCB_NOWAIT)))
goto complete;
return false;
case -EOPNOTSUPP:
/*
* For file systems returning error -EOPNOTSUPP, handle
* IOCB_NOWAIT error case separately and retry without
* IOCB_NOWAIT.
*/
if ((ki_flags & IOCB_NOWAIT))
return false;
break;
}
complete:
nvmet_file_io_done(&req->f.iocb, ret, 0);
return true;
}
static void nvmet_file_buffered_io_work(struct work_struct *w)
{
struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);
nvmet_file_execute_rw(req);
nvmet_file_execute_io(req, 0);
}
static void nvmet_file_execute_rw_buffered_io(struct nvmet_req *req)
static void nvmet_file_submit_buffered_io(struct nvmet_req *req)
{
INIT_WORK(&req->f.work, nvmet_file_buffered_io_work);
queue_work(buffered_io_wq, &req->f.work);
}
static void nvmet_file_execute_rw(struct nvmet_req *req)
{
ssize_t nr_bvec = DIV_ROUND_UP(req->data_len, PAGE_SIZE);
if (!req->sg_cnt || !nr_bvec) {
nvmet_req_complete(req, 0);
return;
}
if (nr_bvec > NVMET_MAX_INLINE_BIOVEC)
req->f.bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec),
GFP_KERNEL);
else
req->f.bvec = req->inline_bvec;
if (unlikely(!req->f.bvec)) {
/* fallback under memory pressure */
req->f.bvec = mempool_alloc(req->ns->bvec_pool, GFP_KERNEL);
req->f.mpool_alloc = true;
} else
req->f.mpool_alloc = false;
if (req->ns->buffered_io) {
if (likely(!req->f.mpool_alloc) &&
nvmet_file_execute_io(req, IOCB_NOWAIT))
return;
nvmet_file_submit_buffered_io(req);
} else
nvmet_file_execute_io(req, 0);
}
u16 nvmet_file_flush(struct nvmet_req *req)
{
if (vfs_fsync(req->ns->file, 1) < 0)
@ -320,10 +361,7 @@ u16 nvmet_file_parse_io_cmd(struct nvmet_req *req)
switch (cmd->common.opcode) {
case nvme_cmd_read:
case nvme_cmd_write:
if (req->ns->buffered_io)
req->execute = nvmet_file_execute_rw_buffered_io;
else
req->execute = nvmet_file_execute_rw;
req->execute = nvmet_file_execute_rw;
req->data_len = nvmet_rw_len(req);
return 0;
case nvme_cmd_flush: