openkylin
/
platform_kernel-5.15
mirror of https://gitee.com/openkylin/platform_kernel-5.15.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

NVMe: Optimise memory usage for I/Os between 4k and 128k 

Add a second memory pool for smaller I/Os.  We can pack 16 of these on a
single page instead of using an entire page for each one.

Signed-off-by: Matthew Wilcox <matthew.r.wilcox@intel.com>
This commit is contained in:
Matthew Wilcox 2011-02-10 10:30:34 -05:00
parent 091b609258
commit 99802a7aee
1 changed files with 23 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
drivers/block/nvme.c
View File

@ -58,6 +58,7 @@ struct nvme_dev {
u32 __iomem *dbs; u32 __iomem *dbs;
struct pci_dev *pci_dev; struct pci_dev *pci_dev;
struct dma_pool *prp_page_pool; struct dma_pool *prp_page_pool;
struct dma_pool *prp_small_pool;
int instance; int instance;
int queue_count; int queue_count;
u32 ctrl_config; u32 ctrl_config;
@ -249,11 +250,6 @@ static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
return 0; return 0;
} }
static __le64 *alloc_prp_list(struct nvme_dev *dev, dma_addr_t *addr)
{
return dma_pool_alloc(dev->prp_page_pool, GFP_ATOMIC, addr);
}
struct nvme_prps { struct nvme_prps {
int npages; int npages;
dma_addr_t first_dma; dma_addr_t first_dma;
@ -271,6 +267,9 @@ static void nvme_free_prps(struct nvme_queue *nvmeq, struct nvme_prps *prps)
return; return;
prp_dma = prps->first_dma; prp_dma = prps->first_dma;
if (prps->npages == 0)
dma_pool_free(dev->prp_small_pool, prps->list[0], prp_dma);
for (i = 0; i < prps->npages; i++) { for (i = 0; i < prps->npages; i++) {
__le64 *prp_list = prps->list[i]; __le64 *prp_list = prps->list[i];
dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
@ -322,6 +321,7 @@ static struct nvme_prps *nvme_setup_prps(struct nvme_queue *nvmeq,
struct scatterlist *sg, int length) struct scatterlist *sg, int length)
{ {
struct nvme_dev *dev = nvmeq->dev; struct nvme_dev *dev = nvmeq->dev;
struct dma_pool *pool;
int dma_len = sg_dma_len(sg); int dma_len = sg_dma_len(sg);
u64 dma_addr = sg_dma_address(sg); u64 dma_addr = sg_dma_address(sg);
int offset = offset_in_page(dma_addr); int offset = offset_in_page(dma_addr);
@ -352,9 +352,16 @@ static struct nvme_prps *nvme_setup_prps(struct nvme_queue *nvmeq,
nprps = DIV_ROUND_UP(length, PAGE_SIZE); nprps = DIV_ROUND_UP(length, PAGE_SIZE);
npages = DIV_ROUND_UP(8 * nprps, PAGE_SIZE); npages = DIV_ROUND_UP(8 * nprps, PAGE_SIZE);
prps = kmalloc(sizeof(*prps) + sizeof(__le64 *) * npages, GFP_ATOMIC); prps = kmalloc(sizeof(*prps) + sizeof(__le64 *) * npages, GFP_ATOMIC);
prps->npages = npages;
prp_page = 0; prp_page = 0;
prp_list = alloc_prp_list(dev, &prp_dma); if (nprps <= (256 / 8)) {
pool = dev->prp_small_pool;
prps->npages = 0;
} else {
pool = dev->prp_page_pool;
prps->npages = npages;
}
prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
prps->list[prp_page++] = prp_list; prps->list[prp_page++] = prp_list;
prps->first_dma = prp_dma; prps->first_dma = prp_dma;
cmd->prp2 = cpu_to_le64(prp_dma); cmd->prp2 = cpu_to_le64(prp_dma);
@ -362,7 +369,7 @@ static struct nvme_prps *nvme_setup_prps(struct nvme_queue *nvmeq,
for (;;) { for (;;) {
if (i == PAGE_SIZE / 8 - 1) { if (i == PAGE_SIZE / 8 - 1) {
__le64 *old_prp_list = prp_list; __le64 *old_prp_list = prp_list;
prp_list = alloc_prp_list(dev, &prp_dma); prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
prps->list[prp_page++] = prp_list; prps->list[prp_page++] = prp_list;
old_prp_list[i] = cpu_to_le64(prp_dma); old_prp_list[i] = cpu_to_le64(prp_dma);
i = 0; i = 0;
@ -1313,12 +1320,20 @@ static int nvme_setup_prp_pools(struct nvme_dev *dev)
if (!dev->prp_page_pool) if (!dev->prp_page_pool)
return -ENOMEM; return -ENOMEM;
/* Optimisation for I/Os between 4k and 128k */
dev->prp_small_pool = dma_pool_create("prp list 256", dmadev,
256, 256, 0);
if (!dev->prp_small_pool) {
dma_pool_destroy(dev->prp_page_pool);
return -ENOMEM;
}
return 0; return 0;
} }
static void nvme_release_prp_pools(struct nvme_dev *dev) static void nvme_release_prp_pools(struct nvme_dev *dev)
{ {
dma_pool_destroy(dev->prp_page_pool); dma_pool_destroy(dev->prp_page_pool);
dma_pool_destroy(dev->prp_small_pool);
} }
/* XXX: Use an ida or something to let remove / add work correctly */ /* XXX: Use an ida or something to let remove / add work correctly */