p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

blk-mq: factor out a "classic" poll helper 

Factor the code to do the classic full metal polling out of blk_poll into
a separate blk_mq_poll_classic helper.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Tested-by: Mark Wunderlich <mark.wunderlich@intel.com>
Link: https://lore.kernel.org/r/20211012111226.760968-7-hch@lst.de
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Christoph Hellwig 2021-10-12 13:12:16 +02:00 committed by Jens Axboe
parent f70299f0d5
commit c6699d6fe0
1 changed files with 56 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
block/blk-mq.c
View File

@ -71,6 +71,14 @@ static inline struct blk_mq_hw_ctx *blk_qc_to_hctx(struct request_queue *q,
return q->queue_hw_ctx[(qc & ~BLK_QC_T_INTERNAL) >> BLK_QC_T_SHIFT];
}
static inline struct request *blk_qc_to_rq(struct blk_mq_hw_ctx *hctx,
blk_qc_t qc)
{
if (blk_qc_t_is_internal(qc))
return blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(qc));
return blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(qc));
}
/*
* Check if any of the ctx, dispatch list or elevator
* have pending work in this hardware queue.
@ -3975,15 +3983,20 @@ static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
return ret;
}
static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
struct request *rq)
static bool blk_mq_poll_hybrid(struct request_queue *q, blk_qc_t qc)
{
struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, qc);
struct request *rq = blk_qc_to_rq(hctx, qc);
struct hrtimer_sleeper hs;
enum hrtimer_mode mode;
unsigned int nsecs;
ktime_t kt;
if (rq->rq_flags & RQF_MQ_POLL_SLEPT)
/*
* If a request has completed on queue that uses an I/O scheduler, we
* won't get back a request from blk_qc_to_rq.
*/
if (!rq || (rq->rq_flags & RQF_MQ_POLL_SLEPT))
return false;
/*
@ -4025,32 +4038,48 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
__set_current_state(TASK_RUNNING);
destroy_hrtimer_on_stack(&hs.timer);
/*
* If we sleep, have the caller restart the poll loop to reset the
* state. Like for the other success return cases, the caller is
* responsible for checking if the IO completed. If the IO isn't
* complete, we'll get called again and will go straight to the busy
* poll loop.
*/
return true;
}
static bool blk_mq_poll_hybrid(struct request_queue *q,
struct blk_mq_hw_ctx *hctx, blk_qc_t cookie)
static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
bool spin)
{
struct request *rq;
struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, cookie);
long state = get_current_state();
int ret;
if (q->poll_nsec == BLK_MQ_POLL_CLASSIC)
return false;
hctx->poll_considered++;
if (!blk_qc_t_is_internal(cookie))
rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
else {
rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
/*
* With scheduling, if the request has completed, we'll
* get a NULL return here, as we clear the sched tag when
* that happens. The request still remains valid, like always,
* so we should be safe with just the NULL check.
*/
if (!rq)
return false;
}
do {
hctx->poll_invoked++;
return blk_mq_poll_hybrid_sleep(q, rq);
ret = q->mq_ops->poll(hctx);
if (ret > 0) {
hctx->poll_success++;
__set_current_state(TASK_RUNNING);
return ret;
}
if (signal_pending_state(state, current))
__set_current_state(TASK_RUNNING);
if (task_is_running(current))
return 1;
if (ret < 0 || !spin)
break;
cpu_relax();
} while (!need_resched());
__set_current_state(TASK_RUNNING);
return 0;
}
/**
@ -4067,9 +4096,6 @@ static bool blk_mq_poll_hybrid(struct request_queue *q,
*/
int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin)
{
struct blk_mq_hw_ctx *hctx;
unsigned int state;
if (!blk_qc_t_valid(cookie) ||
!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
return 0;
@ -4077,46 +4103,12 @@ int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin)
if (current->plug)
blk_flush_plug_list(current->plug, false);
hctx = blk_qc_to_hctx(q, cookie);
/*
* If we sleep, have the caller restart the poll loop to reset
* the state. Like for the other success return cases, the
* caller is responsible for checking if the IO completed. If
* the IO isn't complete, we'll get called again and will go
* straight to the busy poll loop. If specified not to spin,
* we also should not sleep.
*/
if (spin && blk_mq_poll_hybrid(q, hctx, cookie))
return 1;
hctx->poll_considered++;
state = get_current_state();
do {
int ret;
hctx->poll_invoked++;
ret = q->mq_ops->poll(hctx);
if (ret > 0) {
hctx->poll_success++;
__set_current_state(TASK_RUNNING);
return ret;
}
if (signal_pending_state(state, current))
__set_current_state(TASK_RUNNING);
if (task_is_running(current))
/* If specified not to spin, we also should not sleep. */
if (spin && q->poll_nsec != BLK_MQ_POLL_CLASSIC) {
if (blk_mq_poll_hybrid(q, cookie))
return 1;
if (ret < 0 || !spin)
break;
cpu_relax();
} while (!need_resched());
__set_current_state(TASK_RUNNING);
return 0;
}
return blk_mq_poll_classic(q, cookie, spin);
}
EXPORT_SYMBOL_GPL(blk_poll);