for-5.12/block-ipi-2021-02-21
-----BEGIN PGP SIGNATURE----- iQJEBAABCAAuFiEEwPw5LcreJtl1+l5K99NY+ylx4KYFAmAy7bkQHGF4Ym9lQGtl cm5lbC5kawAKCRD301j7KXHgpvZKD/wIWL0IqWiR0RmRPrNRuQ7yVXRVKURRpevR dEIgsAbtRO9RbAxL8RgIBrd4GqRdZFcLqyKwERDbQn/Zx+fEVGs7xeSLcsKGhx/T yLFWuQPITNo5CFj0wMIgzmqB89a8G+GEBm4cv5rG5HsVNg52nDJPpgMNC8KO1THN 8GiXnXsffKTIAGDubR3BSBR+6Z110HhCEJtazGw7f3OtPDV2m81p8F4aWei8vszT DA1nyhZv7a++9xMD92l2BUW366PDOnHEyasL9IAZ46mT7xdIG8j8259ms3I8W++B VuKo5VttJHxdyayK/OoNPjdEDJJVKx3fPZIA57JVFw9yJGGRBefKDPpUZfCr5N3t goVMNexD0zW+voKj9iDlG6310ZXT71RKSeOhBAhaEhXA/GpyxAl//D0rhUF+xlZH 5BcfBafr5yluan7FZCtautoNo0SDwjPdq9zXGKbNFIInSz+ERUC+h/7SZVb52aPp sJJH2aIk7ah5twfIPMwwET2kWslnk1pCo4MQiBghHpp1azAXRSSQ7f5KALGY9NbF al9Q8s2cAYMUN+hLpRTZEBeWWVBQcEcy+T0IUVQo5NkxOFe1lM0G0lfy9mNRHSMP hAeQM9fQ5+x/ALmB9rTya46BrLma1o72DUVoslmpRs7SEZXEdqfqKMjdajLlja2H 4rDnsTfDDg== =W1uK -----END PGP SIGNATURE----- Merge tag 'for-5.12/block-ipi-2021-02-21' of git://git.kernel.dk/linux-block Pull block IPI updates from Jens Axboe: "Avoid IRQ locking for the block IPI handling (Sebastian Andrzej Siewior)" * tag 'for-5.12/block-ipi-2021-02-21' of git://git.kernel.dk/linux-block: blk-mq: Use llist_head for blk_cpu_done blk-mq: Always complete remote completions requests in softirq smp: Process pending softirqs in flush_smp_call_function_from_idle()
This commit is contained in:
Linus Torvalds
commit
ae42c3173b
107
block/blk-mq.c
107
block/blk-mq.c
|
|
@ -41,7 +41,7 @@
|
|||
#include "blk-mq-sched.h"
|
||||
#include "blk-rq-qos.h"
|
||||
|
||||
static DEFINE_PER_CPU(struct list_head, blk_cpu_done);
|
||||
static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
|
||||
|
||||
static void blk_mq_poll_stats_start(struct request_queue *q);
|
||||
static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);
|
||||
|
|
@ -567,80 +567,29 @@ void blk_mq_end_request(struct request *rq, blk_status_t error)
|
|||
}
|
||||
EXPORT_SYMBOL(blk_mq_end_request);
|
||||
|
||||
/*
|
||||
* Softirq action handler - move entries to local list and loop over them
|
||||
* while passing them to the queue registered handler.
|
||||
*/
|
||||
static __latent_entropy void blk_done_softirq(struct softirq_action *h)
|
||||
static void blk_complete_reqs(struct llist_head *list)
|
||||
{
|
||||
struct list_head *cpu_list, local_list;
|
||||
struct llist_node *entry = llist_reverse_order(llist_del_all(list));
|
||||
struct request *rq, *next;
|
||||
|
||||
local_irq_disable();
|
||||
cpu_list = this_cpu_ptr(&blk_cpu_done);
|
||||
list_replace_init(cpu_list, &local_list);
|
||||
local_irq_enable();
|
||||
|
||||
while (!list_empty(&local_list)) {
|
||||
struct request *rq;
|
||||
|
||||
rq = list_entry(local_list.next, struct request, ipi_list);
|
||||
list_del_init(&rq->ipi_list);
|
||||
llist_for_each_entry_safe(rq, next, entry, ipi_list)
|
||||
rq->q->mq_ops->complete(rq);
|
||||
}
|
||||
}
|
||||
|
||||
static void blk_mq_trigger_softirq(struct request *rq)
|
||||
static __latent_entropy void blk_done_softirq(struct softirq_action *h)
|
||||
{
|
||||
struct list_head *list;
|
||||
unsigned long flags;
|
||||
|
||||
local_irq_save(flags);
|
||||
list = this_cpu_ptr(&blk_cpu_done);
|
||||
list_add_tail(&rq->ipi_list, list);
|
||||
|
||||
/*
|
||||
* If the list only contains our just added request, signal a raise of
|
||||
* the softirq. If there are already entries there, someone already
|
||||
* raised the irq but it hasn't run yet.
|
||||
*/
|
||||
if (list->next == &rq->ipi_list)
|
||||
raise_softirq_irqoff(BLOCK_SOFTIRQ);
|
||||
local_irq_restore(flags);
|
||||
blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
|
||||
}
|
||||
|
||||
static int blk_softirq_cpu_dead(unsigned int cpu)
|
||||
{
|
||||
/*
|
||||
* If a CPU goes away, splice its entries to the current CPU
|
||||
* and trigger a run of the softirq
|
||||
*/
|
||||
local_irq_disable();
|
||||
list_splice_init(&per_cpu(blk_cpu_done, cpu),
|
||||
this_cpu_ptr(&blk_cpu_done));
|
||||
raise_softirq_irqoff(BLOCK_SOFTIRQ);
|
||||
local_irq_enable();
|
||||
|
||||
blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static void __blk_mq_complete_request_remote(void *data)
|
||||
{
|
||||
struct request *rq = data;
|
||||
|
||||
/*
|
||||
* For most of single queue controllers, there is only one irq vector
|
||||
* for handling I/O completion, and the only irq's affinity is set
|
||||
* to all possible CPUs. On most of ARCHs, this affinity means the irq
|
||||
* is handled on one specific CPU.
|
||||
*
|
||||
* So complete I/O requests in softirq context in case of single queue
|
||||
* devices to avoid degrading I/O performance due to irqsoff latency.
|
||||
*/
|
||||
if (rq->q->nr_hw_queues == 1)
|
||||
blk_mq_trigger_softirq(rq);
|
||||
else
|
||||
rq->q->mq_ops->complete(rq);
|
||||
__raise_softirq_irqoff(BLOCK_SOFTIRQ);
|
||||
}
|
||||
|
||||
static inline bool blk_mq_complete_need_ipi(struct request *rq)
|
||||
|
|
@ -669,6 +618,30 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq)
|
|||
return cpu_online(rq->mq_ctx->cpu);
|
||||
}
|
||||
|
||||
static void blk_mq_complete_send_ipi(struct request *rq)
|
||||
{
|
||||
struct llist_head *list;
|
||||
unsigned int cpu;
|
||||
|
||||
cpu = rq->mq_ctx->cpu;
|
||||
list = &per_cpu(blk_cpu_done, cpu);
|
||||
if (llist_add(&rq->ipi_list, list)) {
|
||||
INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq);
|
||||
smp_call_function_single_async(cpu, &rq->csd);
|
||||
}
|
||||
}
|
||||
|
||||
static void blk_mq_raise_softirq(struct request *rq)
|
||||
{
|
||||
struct llist_head *list;
|
||||
|
||||
preempt_disable();
|
||||
list = this_cpu_ptr(&blk_cpu_done);
|
||||
if (llist_add(&rq->ipi_list, list))
|
||||
raise_softirq(BLOCK_SOFTIRQ);
|
||||
preempt_enable();
|
||||
}
|
||||
|
||||
bool blk_mq_complete_request_remote(struct request *rq)
|
||||
{
|
||||
WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
|
||||
|
|
@ -681,16 +654,16 @@ bool blk_mq_complete_request_remote(struct request *rq)
|
|||
return false;
|
||||
|
||||
if (blk_mq_complete_need_ipi(rq)) {
|
||||
INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq);
|
||||
smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd);
|
||||
} else {
|
||||
if (rq->q->nr_hw_queues > 1)
|
||||
return false;
|
||||
blk_mq_trigger_softirq(rq);
|
||||
blk_mq_complete_send_ipi(rq);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (rq->q->nr_hw_queues == 1) {
|
||||
blk_mq_raise_softirq(rq);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
|
||||
|
||||
/**
|
||||
|
|
@ -3957,7 +3930,7 @@ static int __init blk_mq_init(void)
|
|||
int i;
|
||||
|
||||
for_each_possible_cpu(i)
|
||||
INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i));
|
||||
init_llist_head(&per_cpu(blk_cpu_done, i));
|
||||
open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
|
||||
|
||||
cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ struct request {
|
|||
*/
|
||||
union {
|
||||
struct hlist_node hash; /* merge hash */
|
||||
struct list_head ipi_list;
|
||||
struct llist_node ipi_list;
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
|
|||
Loading…
Reference in New Issue