From 4e89dce725213d3d0b0475211b500eda4ef4bf2f Mon Sep 17 00:00:00 2001 From: Vijayanand Jitta Date: Wed, 30 Sep 2020 13:14:23 +0530 Subject: [PATCH 1/6] iommu/iova: Retry from last rb tree node if iova search fails When ever a new iova alloc request comes iova is always searched from the cached node and the nodes which are previous to cached node. So, even if there is free iova space available in the nodes which are next to the cached node iova allocation can still fail because of this approach. Consider the following sequence of iova alloc and frees on 1GB of iova space 1) alloc - 500MB 2) alloc - 12MB 3) alloc - 499MB 4) free - 12MB which was allocated in step 2 5) alloc - 13MB After the above sequence we will have 12MB of free iova space and cached node will be pointing to the iova pfn of last alloc of 13MB which will be the lowest iova pfn of that iova space. Now if we get an alloc request of 2MB we just search from cached node and then look for lower iova pfn's for free iova and as they aren't any, iova alloc fails though there is 12MB of free iova space. To avoid such iova search failures do a retry from the last rb tree node when iova search fails, this will search the entire tree and get an iova if its available. Signed-off-by: Vijayanand Jitta Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/1601451864-5956-1-git-send-email-vjitta@codeaurora.org Signed-off-by: Will Deacon --- drivers/iommu/iova.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 30d969a4c5fd..c3a1a8e6f2c1 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -184,8 +184,9 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad, struct rb_node *curr, *prev; struct iova *curr_iova; unsigned long flags; - unsigned long new_pfn; + unsigned long new_pfn, retry_pfn; unsigned long align_mask = ~0UL; + unsigned long high_pfn = limit_pfn, low_pfn = iovad->start_pfn; if (size_aligned) align_mask <<= fls_long(size - 1); @@ -198,15 +199,25 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad, curr = __get_cached_rbnode(iovad, limit_pfn); curr_iova = rb_entry(curr, struct iova, node); + retry_pfn = curr_iova->pfn_hi + 1; + +retry: do { - limit_pfn = min(limit_pfn, curr_iova->pfn_lo); - new_pfn = (limit_pfn - size) & align_mask; + high_pfn = min(high_pfn, curr_iova->pfn_lo); + new_pfn = (high_pfn - size) & align_mask; prev = curr; curr = rb_prev(curr); curr_iova = rb_entry(curr, struct iova, node); - } while (curr && new_pfn <= curr_iova->pfn_hi); + } while (curr && new_pfn <= curr_iova->pfn_hi && new_pfn >= low_pfn); - if (limit_pfn < size || new_pfn < iovad->start_pfn) { + if (high_pfn < size || new_pfn < low_pfn) { + if (low_pfn == iovad->start_pfn && retry_pfn < limit_pfn) { + high_pfn = limit_pfn; + low_pfn = retry_pfn; + curr = &iovad->anchor.node; + curr_iova = rb_entry(curr, struct iova, node); + goto retry; + } iovad->max32_alloc_size = size; goto iova32_full; } From 6fa3525b455ae1fde5b424907141b33651f137b0 Mon Sep 17 00:00:00 2001 From: Vijayanand Jitta Date: Wed, 30 Sep 2020 13:14:24 +0530 Subject: [PATCH 2/6] iommu/iova: Free global iova rcache on iova alloc failure When ever an iova alloc request fails we free the iova ranges present in the percpu iova rcaches and then retry but the global iova rcache is not freed as a result we could still see iova alloc failure even after retry as global rcache is holding the iova's which can cause fragmentation. So, free the global iova rcache as well and then go for the retry. Signed-off-by: Vijayanand Jitta Reviewed-by: Robin Murphy Acked-by: John Garry Link: https://lore.kernel.org/r/1601451864-5956-2-git-send-email-vjitta@codeaurora.org Signed-off-by: Will Deacon --- drivers/iommu/iova.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index c3a1a8e6f2c1..ea04a88c673d 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -25,6 +25,7 @@ static void init_iova_rcaches(struct iova_domain *iovad); static void free_iova_rcaches(struct iova_domain *iovad); static void fq_destroy_all_entries(struct iova_domain *iovad); static void fq_flush_timeout(struct timer_list *t); +static void free_global_cached_iovas(struct iova_domain *iovad); void init_iova_domain(struct iova_domain *iovad, unsigned long granule, @@ -442,6 +443,7 @@ alloc_iova_fast(struct iova_domain *iovad, unsigned long size, flush_rcache = false; for_each_online_cpu(cpu) free_cpu_cached_iovas(cpu, iovad); + free_global_cached_iovas(iovad); goto retry; } @@ -1057,5 +1059,25 @@ void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad) } } +/* + * free all the IOVA ranges of global cache + */ +static void free_global_cached_iovas(struct iova_domain *iovad) +{ + struct iova_rcache *rcache; + unsigned long flags; + int i, j; + + for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) { + rcache = &iovad->rcaches[i]; + spin_lock_irqsave(&rcache->lock, flags); + for (j = 0; j < rcache->depot_size; ++j) { + iova_magazine_free_pfns(rcache->depot[j], iovad); + iova_magazine_free(rcache->depot[j]); + } + rcache->depot_size = 0; + spin_unlock_irqrestore(&rcache->lock, flags); + } +} MODULE_AUTHOR("Anil S Keshavamurthy "); MODULE_LICENSE("GPL"); From 3a651b3a27a1ee35879499ead3942dc854a20968 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Tue, 17 Nov 2020 18:25:34 +0800 Subject: [PATCH 3/6] iommu: avoid taking iova_rbtree_lock twice Both find_iova() and __free_iova() take iova_rbtree_lock, there is no reason to take and release it twice inside free_iova(). Fold them into one critical section by calling the unlock versions instead. Signed-off-by: Cong Wang Reviewed-by: Robin Murphy Signed-off-by: John Garry Link: https://lore.kernel.org/r/1605608734-84416-5-git-send-email-john.garry@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/iova.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index ea04a88c673d..ff59d8aa3041 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -402,10 +402,14 @@ EXPORT_SYMBOL_GPL(__free_iova); void free_iova(struct iova_domain *iovad, unsigned long pfn) { - struct iova *iova = find_iova(iovad, pfn); + unsigned long flags; + struct iova *iova; + spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); + iova = private_find_iova(iovad, pfn); if (iova) - __free_iova(iovad, iova); + private_free_iova(iovad, iova); + spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); } EXPORT_SYMBOL_GPL(free_iova); From 2f24dfb71208eeb0174f08dd56ca6d3c17b279a5 Mon Sep 17 00:00:00 2001 From: John Garry Date: Fri, 4 Dec 2020 02:34:50 +0800 Subject: [PATCH 4/6] iommu: Delete split_and_remove_iova() Function split_and_remove_iova() has not been referenced since commit e70b081c6f37 ("iommu/vt-d: Remove IOVA handling code from the non-dma_ops path"), so delete it. Signed-off-by: John Garry Link: https://lore.kernel.org/r/1607020492-189471-2-git-send-email-john.garry@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/iova.c | 41 ----------------------------------------- include/linux/iova.h | 10 ---------- 2 files changed, 51 deletions(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index ff59d8aa3041..3331c040b2b0 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -742,47 +742,6 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to) } EXPORT_SYMBOL_GPL(copy_reserved_iova); -struct iova * -split_and_remove_iova(struct iova_domain *iovad, struct iova *iova, - unsigned long pfn_lo, unsigned long pfn_hi) -{ - unsigned long flags; - struct iova *prev = NULL, *next = NULL; - - spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); - if (iova->pfn_lo < pfn_lo) { - prev = alloc_and_init_iova(iova->pfn_lo, pfn_lo - 1); - if (prev == NULL) - goto error; - } - if (iova->pfn_hi > pfn_hi) { - next = alloc_and_init_iova(pfn_hi + 1, iova->pfn_hi); - if (next == NULL) - goto error; - } - - __cached_rbnode_delete_update(iovad, iova); - rb_erase(&iova->node, &iovad->rbroot); - - if (prev) { - iova_insert_rbtree(&iovad->rbroot, prev, NULL); - iova->pfn_lo = pfn_lo; - } - if (next) { - iova_insert_rbtree(&iovad->rbroot, next, NULL); - iova->pfn_hi = pfn_hi; - } - spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); - - return iova; - -error: - spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); - if (prev) - free_iova_mem(prev); - return NULL; -} - /* * Magazine caches for IOVA ranges. For an introduction to magazines, * see the USENIX 2001 paper "Magazines and Vmem: Extending the Slab diff --git a/include/linux/iova.h b/include/linux/iova.h index a0637abffee8..a77add571c50 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -160,8 +160,6 @@ int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb, iova_entry_dtor entry_dtor); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); -struct iova *split_and_remove_iova(struct iova_domain *iovad, - struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi); void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); #else static inline int iova_cache_get(void) @@ -258,14 +256,6 @@ static inline void put_iova_domain(struct iova_domain *iovad) { } -static inline struct iova *split_and_remove_iova(struct iova_domain *iovad, - struct iova *iova, - unsigned long pfn_lo, - unsigned long pfn_hi) -{ - return NULL; -} - static inline void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad) { From 51b70b817b187e48155fc492adb9ce80bdb21b70 Mon Sep 17 00:00:00 2001 From: John Garry Date: Fri, 4 Dec 2020 02:34:51 +0800 Subject: [PATCH 5/6] iommu: Stop exporting alloc_iova_mem() It is not used outside iova.c Signed-off-by: John Garry Link: https://lore.kernel.org/r/1607020492-189471-3-git-send-email-john.garry@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/iova.c | 3 +-- include/linux/iova.h | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 3331c040b2b0..bb5cb67ee311 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -243,11 +243,10 @@ static struct kmem_cache *iova_cache; static unsigned int iova_cache_users; static DEFINE_MUTEX(iova_cache_mutex); -struct iova *alloc_iova_mem(void) +static struct iova *alloc_iova_mem(void) { return kmem_cache_zalloc(iova_cache, GFP_ATOMIC | __GFP_NOWARN); } -EXPORT_SYMBOL(alloc_iova_mem); void free_iova_mem(struct iova *iova) { diff --git a/include/linux/iova.h b/include/linux/iova.h index a77add571c50..d8c011b6d4ed 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -136,7 +136,6 @@ static inline unsigned long iova_pfn(struct iova_domain *iovad, dma_addr_t iova) int iova_cache_get(void); void iova_cache_put(void); -struct iova *alloc_iova_mem(void); void free_iova_mem(struct iova *iova); void free_iova(struct iova_domain *iovad, unsigned long pfn); void __free_iova(struct iova_domain *iovad, struct iova *iova); @@ -171,11 +170,6 @@ static inline void iova_cache_put(void) { } -static inline struct iova *alloc_iova_mem(void) -{ - return NULL; -} - static inline void free_iova_mem(struct iova *iova) { } From 176cfc187c24287a363e7612cd2385ba40c2042b Mon Sep 17 00:00:00 2001 From: John Garry Date: Fri, 4 Dec 2020 02:34:52 +0800 Subject: [PATCH 6/6] iommu: Stop exporting free_iova_mem() It has no user outside iova.c Signed-off-by: John Garry Link: https://lore.kernel.org/r/1607020492-189471-4-git-send-email-john.garry@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/iova.c | 3 +-- include/linux/iova.h | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index bb5cb67ee311..4bb3293ae4d7 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -248,12 +248,11 @@ static struct iova *alloc_iova_mem(void) return kmem_cache_zalloc(iova_cache, GFP_ATOMIC | __GFP_NOWARN); } -void free_iova_mem(struct iova *iova) +static void free_iova_mem(struct iova *iova) { if (iova->pfn_lo != IOVA_ANCHOR) kmem_cache_free(iova_cache, iova); } -EXPORT_SYMBOL(free_iova_mem); int iova_cache_get(void) { diff --git a/include/linux/iova.h b/include/linux/iova.h index d8c011b6d4ed..76e16ae20729 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -136,7 +136,6 @@ static inline unsigned long iova_pfn(struct iova_domain *iovad, dma_addr_t iova) int iova_cache_get(void); void iova_cache_put(void); -void free_iova_mem(struct iova *iova); void free_iova(struct iova_domain *iovad, unsigned long pfn); void __free_iova(struct iova_domain *iovad, struct iova *iova); struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size, @@ -170,10 +169,6 @@ static inline void iova_cache_put(void) { } -static inline void free_iova_mem(struct iova *iova) -{ -} - static inline void free_iova(struct iova_domain *iovad, unsigned long pfn) { }