From 7d30a621028a22a9e8fbae5c5d8dc455869e4999 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 29 Nov 2019 00:11:53 +0300 Subject: [PATCH 1/9] blk-mq: optimise rq sort function Check "!=" in multi-layer comparisons. The same memory usage, fewer instructions, and 2 from 4 jumps are replaced with SETcc. Note, that list_sort() doesn't differ 0 and <0. Reviewed-by: Nikolay Borisov Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- block/blk-mq.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 323c9cb28066..f32a3cfdd34e 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1668,14 +1668,10 @@ static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b) struct request *rqa = container_of(a, struct request, queuelist); struct request *rqb = container_of(b, struct request, queuelist); - if (rqa->mq_ctx < rqb->mq_ctx) - return -1; - else if (rqa->mq_ctx > rqb->mq_ctx) - return 1; - else if (rqa->mq_hctx < rqb->mq_hctx) - return -1; - else if (rqa->mq_hctx > rqb->mq_hctx) - return 1; + if (rqa->mq_ctx != rqb->mq_ctx) + return rqa->mq_ctx > rqb->mq_ctx; + if (rqa->mq_hctx != rqb->mq_hctx) + return rqa->mq_hctx > rqb->mq_hctx; return blk_rq_pos(rqa) > blk_rq_pos(rqb); } From 28ca0d6d39ab1d01c86762c82a585b7cedd2920c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 29 Nov 2019 00:11:54 +0300 Subject: [PATCH 2/9] list: introduce list_for_each_continue() As other *continue() helpers, this continues iteration from a given position. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- include/linux/list.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/list.h b/include/linux/list.h index 85c92555e31f..3c391bbd03c3 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -538,6 +538,16 @@ static inline void list_splice_tail_init(struct list_head *list, #define list_for_each(pos, head) \ for (pos = (head)->next; pos != (head); pos = pos->next) +/** + * list_for_each_continue - continue iteration over a list + * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + * + * Continue to iterate over a list, continuing after the current position. + */ +#define list_for_each_continue(pos, head) \ + for (pos = pos->next; pos != (head); pos = pos->next) + /** * list_for_each_prev - iterate over a list backwards * @pos: the &struct list_head to use as a loop cursor. From 95ed0c5b1292fb90fe100af88541a7bef6ba44ec Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 29 Nov 2019 00:11:55 +0300 Subject: [PATCH 3/9] blk-mq: optimise blk_mq_flush_plug_list() Instead of using list_del_init() in a loop, that generates a lot of unnecessary memory read/writes, iterate from the first request of a batch and cut out a sublist with list_cut_before(). Apart from removing the list node initialisation part, this is more register-friendly, and the assembly uses the stack less intensively. list_empty() at the beginning is done with hope, that the compiler can optimise out the same check in the following list_splice_init(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- block/blk-mq.c | 55 +++++++++++++++++--------------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index f32a3cfdd34e..3c71d52b6401 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1678,14 +1678,10 @@ static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b) void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) { - struct blk_mq_hw_ctx *this_hctx; - struct blk_mq_ctx *this_ctx; - struct request_queue *this_q; - struct request *rq; LIST_HEAD(list); - LIST_HEAD(rq_list); - unsigned int depth; + if (list_empty(&plug->mq_list)) + return; list_splice_init(&plug->mq_list, &list); if (plug->rq_count > 2 && plug->multiple_queues) @@ -1693,42 +1689,27 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) plug->rq_count = 0; - this_q = NULL; - this_hctx = NULL; - this_ctx = NULL; - depth = 0; + do { + struct list_head rq_list; + struct request *rq, *head_rq = list_entry_rq(list.next); + struct list_head *pos = &head_rq->queuelist; /* skip first */ + struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx; + struct blk_mq_ctx *this_ctx = head_rq->mq_ctx; + unsigned int depth = 1; - while (!list_empty(&list)) { - rq = list_entry_rq(list.next); - list_del_init(&rq->queuelist); - BUG_ON(!rq->q); - if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx) { - if (this_hctx) { - trace_block_unplug(this_q, depth, !from_schedule); - blk_mq_sched_insert_requests(this_hctx, this_ctx, - &rq_list, - from_schedule); - } - - this_q = rq->q; - this_ctx = rq->mq_ctx; - this_hctx = rq->mq_hctx; - depth = 0; + list_for_each_continue(pos, &list) { + rq = list_entry_rq(pos); + BUG_ON(!rq->q); + if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx) + break; + depth++; } - depth++; - list_add_tail(&rq->queuelist, &rq_list); - } - - /* - * If 'this_hctx' is set, we know we have entries to complete - * on 'rq_list'. Do those. - */ - if (this_hctx) { - trace_block_unplug(this_q, depth, !from_schedule); + list_cut_before(&rq_list, &list, pos); + trace_block_unplug(head_rq->q, depth, !from_schedule); blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list, from_schedule); - } + } while(!list_empty(&list)); } static void blk_mq_bio_to_request(struct request *rq, struct bio *bio, From a754bd5f1874978f55814b4498f66e4a0fd5b256 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 23 Dec 2019 16:13:51 +0800 Subject: [PATCH 4/9] block: Allow t10-pi to be modular Currently t10-pi can only be built into the block layer which via crc-t10dif pulls in a whole chunk of the Crypto API. In fact all users of t10-pi work as modules and there is no reason for it to always be built-in. This patch adds a new hidden option for t10-pi that is selected automatically based on BLK_DEV_INTEGRITY and whether the users of t10-pi are built-in or not. Signed-off-by: Herbert Xu Signed-off-by: Jens Axboe --- block/Kconfig | 6 +++++- block/Makefile | 3 ++- block/t10-pi.c | 3 +++ drivers/nvme/host/Kconfig | 1 + drivers/scsi/Kconfig | 1 + 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/block/Kconfig b/block/Kconfig index c23094a14a2b..3bc76bb113a0 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -66,7 +66,6 @@ config BLK_DEV_BSGLIB config BLK_DEV_INTEGRITY bool "Block layer data integrity support" - select CRC_T10DIF if BLK_DEV_INTEGRITY ---help--- Some storage devices allow extra information to be stored/retrieved to help protect the data. The block layer @@ -77,6 +76,11 @@ config BLK_DEV_INTEGRITY T10/SCSI Data Integrity Field or the T13/ATA External Path Protection. If in doubt, say N. +config BLK_DEV_INTEGRITY_T10 + tristate + depends on BLK_DEV_INTEGRITY + select CRC_T10DIF + config BLK_DEV_ZONED bool "Zoned block device support" select MQ_IOSCHED_DEADLINE diff --git a/block/Makefile b/block/Makefile index 205a5f2fef17..f6cef6d4363c 100644 --- a/block/Makefile +++ b/block/Makefile @@ -27,7 +27,8 @@ obj-$(CONFIG_IOSCHED_BFQ) += bfq.o obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o -obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o +obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o +obj-$(CONFIG_BLK_DEV_INTEGRITY_T10) += t10-pi.o obj-$(CONFIG_BLK_MQ_PCI) += blk-mq-pci.o obj-$(CONFIG_BLK_MQ_VIRTIO) += blk-mq-virtio.o obj-$(CONFIG_BLK_MQ_RDMA) += blk-mq-rdma.o diff --git a/block/t10-pi.c b/block/t10-pi.c index f4907d941f03..d910534b3a41 100644 --- a/block/t10-pi.c +++ b/block/t10-pi.c @@ -7,6 +7,7 @@ #include #include #include +#include #include typedef __be16 (csum_fn) (void *, unsigned int); @@ -280,3 +281,5 @@ const struct blk_integrity_profile t10_pi_type3_ip = { .complete_fn = t10_pi_type3_complete, }; EXPORT_SYMBOL(t10_pi_type3_ip); + +MODULE_LICENSE("GPL"); diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig index c6439638a419..b9358db83e96 100644 --- a/drivers/nvme/host/Kconfig +++ b/drivers/nvme/host/Kconfig @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config NVME_CORE tristate + select BLK_DEV_INTEGRITY_T10 if BLK_DEV_INTEGRITY config BLK_DEV_NVME tristate "NVM Express block device" diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig index 90cf4691b8c3..a7881f8eb05e 100644 --- a/drivers/scsi/Kconfig +++ b/drivers/scsi/Kconfig @@ -68,6 +68,7 @@ comment "SCSI support type (disk, tape, CD-ROM)" config BLK_DEV_SD tristate "SCSI disk support" depends on SCSI + select BLK_DEV_INTEGRITY_T10 if BLK_DEV_INTEGRITY ---help--- If you want to use SCSI hard disks, Fibre Channel disks, Serial ATA (SATA) or Parallel ATA (PATA) hard disks, From 105663f73e71b5e79ecf472c0499c9acde654b28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Mon, 6 Jan 2020 15:08:18 -0300 Subject: [PATCH 5/9] blk-mq: Document functions for sending request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add or improve documentation for function regarding creating and sending IO requests to the hardware. Signed-off-by: André Almeida Signed-off-by: Jens Axboe --- block/blk-mq.c | 85 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 3c71d52b6401..a12b1763508d 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -641,6 +641,14 @@ bool blk_mq_complete_request(struct request *rq) } EXPORT_SYMBOL(blk_mq_complete_request); +/** + * blk_mq_start_request - Start processing a request + * @rq: Pointer to request to be started + * + * Function used by device drivers to notify the block layer that a request + * is going to be processed now, so blk layer can do proper initializations + * such as starting the timeout timer. + */ void blk_mq_start_request(struct request *rq) { struct request_queue *q = rq->q; @@ -1327,6 +1335,12 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, return (queued + errors) != 0; } +/** + * __blk_mq_run_hw_queue - Run a hardware queue. + * @hctx: Pointer to the hardware queue to run. + * + * Send pending requests to the hardware. + */ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) { int srcu_idx; @@ -1424,6 +1438,15 @@ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) return next_cpu; } +/** + * __blk_mq_delay_run_hw_queue - Run (or schedule to run) a hardware queue. + * @hctx: Pointer to the hardware queue to run. + * @async: If we want to run the queue asynchronously. + * @msecs: Microseconds of delay to wait before running the queue. + * + * If !@async, try to run the queue now. Else, run the queue asynchronously and + * with a delay of @msecs. + */ static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, unsigned long msecs) { @@ -1445,12 +1468,28 @@ static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, msecs_to_jiffies(msecs)); } +/** + * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously. + * @hctx: Pointer to the hardware queue to run. + * @msecs: Microseconds of delay to wait before running the queue. + * + * Run a hardware queue asynchronously with a delay of @msecs. + */ void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) { __blk_mq_delay_run_hw_queue(hctx, true, msecs); } EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); +/** + * blk_mq_run_hw_queue - Start to run a hardware queue. + * @hctx: Pointer to the hardware queue to run. + * @async: If we want to run the queue asynchronously. + * + * Check if the request queue is not in a quiesced state and if there are + * pending requests to be sent. If this is true, run the queue to send requests + * to hardware. + */ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) { int srcu_idx; @@ -1474,6 +1513,11 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) } EXPORT_SYMBOL(blk_mq_run_hw_queue); +/** + * blk_mq_run_hw_queue - Run all hardware queues in a request queue. + * @q: Pointer to the request queue to run. + * @async: If we want to run the queue asynchronously. + */ void blk_mq_run_hw_queues(struct request_queue *q, bool async) { struct blk_mq_hw_ctx *hctx; @@ -1625,7 +1669,11 @@ void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, blk_mq_hctx_mark_pending(hctx, ctx); } -/* +/** + * blk_mq_request_bypass_insert - Insert a request at dispatch list. + * @rq: Pointer to request to be inserted. + * @run_queue: If we should run the hardware queue after inserting the request. + * * Should only be used carefully, when the caller knows we want to * bypass a potential IO scheduler on the target device. */ @@ -1805,6 +1853,17 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, return BLK_STS_OK; } +/** + * blk_mq_try_issue_directly - Try to send a request directly to device driver. + * @hctx: Pointer of the associated hardware queue. + * @rq: Pointer to request to be sent. + * @cookie: Request queue cookie. + * + * If the device has enough resources to accept a new request now, send the + * request directly to device driver. Else, insert at hctx->dispatch queue, so + * we can try send it another time in the future. Requests inserted at this + * queue have higher priority. + */ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, struct request *rq, blk_qc_t *cookie) { @@ -1882,6 +1941,22 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) } } +/** + * blk_mq_make_request - Create and send a request to block device. + * @q: Request queue pointer. + * @bio: Bio pointer. + * + * Builds up a request structure from @q and @bio and send to the device. The + * request may not be queued directly to hardware if: + * * This request can be merged with another one + * * We want to place request at plug queue for possible future merging + * * There is an IO scheduler active at this queue + * + * It will not queue the request if there is an error with the bio, or at the + * request creation. + * + * Returns: Request queue cookie. + */ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) { const int is_sync = op_is_sync(bio->bi_opf); @@ -1927,7 +2002,7 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) plug = blk_mq_plug(q, bio); if (unlikely(is_flush_fua)) { - /* bypass scheduler for flush rq */ + /* Bypass scheduler for flush requests */ blk_insert_flush(rq); blk_mq_run_hw_queue(data.hctx, true); } else if (plug && (q->nr_hw_queues == 1 || q->mq_ops->commit_rqs || @@ -1955,6 +2030,7 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) blk_add_rq_to_plug(plug, rq); } else if (q->elevator) { + /* Insert the request at the IO scheduler queue */ blk_mq_sched_insert_request(rq, false, true, true); } else if (plug && !blk_queue_nomerges(q)) { /* @@ -1981,8 +2057,13 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) } } else if ((q->nr_hw_queues > 1 && is_sync) || !data.hctx->dispatch_busy) { + /* + * There is no scheduler and we can try to send directly + * to the hardware. + */ blk_mq_try_issue_directly(data.hctx, rq, &cookie); } else { + /* Default case. */ blk_mq_sched_insert_request(rq, false, true, true); } From 8e42d239cb027143915cae13eb2ecf1360ee24de Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Tue, 7 Jan 2020 13:58:17 -0800 Subject: [PATCH 6/9] block: mark zone-mgmt bios with REQ_SYNC In the current implementation, final zone-mgmt request is issued with submit_bio_wait() which marks the bio REQ_SYNC. This is needed since immediate action is expected for zone-mgmt requests as these are blocking operations. This also bypasses the scheduler in the blk_mq_make_request() and dispatches the request directly into the hw ctx. This patch marks all the chained bios REQ_SYNC so that we can have above-mentioned behavior for non-final bios also. Reviewed-by: Damien Le Moal Reviewed-by: Bob Liu Signed-off-by: Chaitanya Kulkarni Signed-off-by: Jens Axboe --- block/blk-zoned.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/blk-zoned.c b/block/blk-zoned.c index d00fcfd71dfe..05741c6f618b 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -198,7 +198,7 @@ int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op, break; } - bio->bi_opf = op; + bio->bi_opf = op | REQ_SYNC; bio->bi_iter.bi_sector = sector; sector += zone_sectors; From b7f22d993f51c64d79b7fd7ad2c060e3fdc006c0 Mon Sep 17 00:00:00 2001 From: Alex Shi Date: Tue, 21 Jan 2020 16:48:54 +0800 Subject: [PATCH 7/9] block/bfq: remove unused bfq_class_rt which never used This macro is never used after introduced from commit aee69d78dec0 ("block, bfq: introduce the BFQ-v0 I/O scheduler as an extra scheduler") Better to remove it. Signed-off-by: Alex Shi Cc: Paolo Valente Cc: Jens Axboe Cc: linux-block@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Jens Axboe --- block/bfq-iosched.c | 1 - 1 file changed, 1 deletion(-) diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index ad4af4aaf2ce..4686b68b48b4 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -427,7 +427,6 @@ void bfq_schedule_dispatch(struct bfq_data *bfqd) } #define bfq_class_idle(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_IDLE) -#define bfq_class_rt(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_RT) #define bfq_sample_valid(samples) ((samples) > 80) From 554d21efb0d2e102a14f8c4b6bdfbcfbe54b3c65 Mon Sep 17 00:00:00 2001 From: Wen Yang Date: Mon, 20 Jan 2020 18:04:43 +0800 Subject: [PATCH 8/9] block, bfq: improve arithmetic division in bfq_delta() do_div() does a 64-by-32 division. Use div64_ul() instead of it if the divisor is unsigned long, to avoid truncation to 32-bit. And as a nice side effect also cleans up the function a bit. Signed-off-by: Wen Yang Cc: Paolo Valente Cc: Jens Axboe Cc: linux-block@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Jens Axboe --- block/bfq-wf2q.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c index 05f0bf4a1144..ffe9ce9faa89 100644 --- a/block/bfq-wf2q.c +++ b/block/bfq-wf2q.c @@ -277,10 +277,7 @@ struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity) */ static u64 bfq_delta(unsigned long service, unsigned long weight) { - u64 d = (u64)service << WFQ_SERVICE_SHIFT; - - do_div(d, weight); - return d; + return div64_ul((u64)service << WFQ_SERVICE_SHIFT, weight); } /** From 5336da37a5eac761611352707c3890a3cf857aa6 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 23 Jan 2020 00:29:21 +0000 Subject: [PATCH 9/9] partitions/ldm: fix spelling mistake "to" -> "too" There is a spelling mistake in a ldm_error message. Fix it. Signed-off-by: Colin Ian King Signed-off-by: Jens Axboe --- block/partitions/ldm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/partitions/ldm.c b/block/partitions/ldm.c index fe5d970e2e60..a2d97ee1908c 100644 --- a/block/partitions/ldm.c +++ b/block/partitions/ldm.c @@ -1233,7 +1233,7 @@ static bool ldm_frag_add (const u8 *data, int size, struct list_head *frags) BUG_ON (!data || !frags); if (size < 2 * VBLK_SIZE_HEAD) { - ldm_error("Value of size is to small."); + ldm_error("Value of size is too small."); return false; }