p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

userfaultfd: teach vma_merge to merge across vma->vm_userfaultfd_ctx 

vma->vm_userfaultfd_ctx is yet another vma parameter that vma_merge
must be aware about so that we can merge vmas back like they were
originally before arming the userfaultfd on some memory range.

Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Acked-by: Pavel Emelyanov <xemul@parallels.com>
Cc: Sanidhya Kashyap <sanidhya.gatech@gmail.com>
Cc: zhang.zhanghailiang@huawei.com
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Andres Lagar-Cavilla <andreslc@google.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Hugh Dickins <hughd@google.com>
Cc: Peter Feiner <pfeiner@google.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: "Huangpeng (Peter)" <peter.huangpeng@huawei.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Andrea Arcangeli 2015-09-04 15:46:24 -07:00 committed by Linus Torvalds
parent 6b251fc96c
commit 19a809afe2
6 changed files with 36 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/mm.h
View File

@ -1835,7 +1835,7 @@ extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
extern struct vm_area_struct *vma_merge(struct mm_struct *, extern struct vm_area_struct *vma_merge(struct mm_struct *,
struct vm_area_struct *prev, unsigned long addr, unsigned long end, struct vm_area_struct *prev, unsigned long addr, unsigned long end,
unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t, unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
struct mempolicy *); struct mempolicy *, struct vm_userfaultfd_ctx);
extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
extern int split_vma(struct mm_struct *, extern int split_vma(struct mm_struct *,
struct vm_area_struct *, unsigned long addr, int new_below); struct vm_area_struct *, unsigned long addr, int new_below);

3
mm/madvise.c
View File

@ -103,7 +103,8 @@ static long madvise_behavior(struct vm_area_struct *vma,
pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
*prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma, *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
vma->vm_file, pgoff, vma_policy(vma)); vma->vm_file, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx);
if (*prev) { if (*prev) {
vma = *prev; vma = *prev;
goto success; goto success;

4
mm/mempolicy.c
View File

@ -722,8 +722,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
pgoff = vma->vm_pgoff + pgoff = vma->vm_pgoff +
((vmstart - vma->vm_start) >> PAGE_SHIFT); ((vmstart - vma->vm_start) >> PAGE_SHIFT);
prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags, prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
vma->anon_vma, vma->vm_file, pgoff, vma->anon_vma, vma->vm_file, pgoff,
new_pol); new_pol, vma->vm_userfaultfd_ctx);
if (prev) { if (prev) {
vma = prev; vma = prev;
next = vma->vm_next; next = vma->vm_next;

3
mm/mlock.c
View File

@ -510,7 +510,8 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev,
pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
*prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma, *prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma,
vma->vm_file, pgoff, vma_policy(vma)); vma->vm_file, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx);
if (*prev) { if (*prev) {
vma = *prev; vma = *prev;
goto success; goto success;

40
mm/mmap.c
View File

@ -41,6 +41,7 @@
#include <linux/notifier.h> #include <linux/notifier.h>
#include <linux/memory.h> #include <linux/memory.h>
#include <linux/printk.h> #include <linux/printk.h>
#include <linux/userfaultfd_k.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
@ -919,7 +920,8 @@ again: remove_next = 1 + (end > next->vm_end);
* per-vma resources, so we don't attempt to merge those. * per-vma resources, so we don't attempt to merge those.
*/ */
static inline int is_mergeable_vma(struct vm_area_struct *vma, static inline int is_mergeable_vma(struct vm_area_struct *vma,
struct file *file, unsigned long vm_flags) struct file *file, unsigned long vm_flags,
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
{ {
/* /*
* VM_SOFTDIRTY should not prevent from VMA merging, if we * VM_SOFTDIRTY should not prevent from VMA merging, if we
@ -935,6 +937,8 @@ static inline int is_mergeable_vma(struct vm_area_struct *vma,
return 0; return 0;
if (vma->vm_ops && vma->vm_ops->close) if (vma->vm_ops && vma->vm_ops->close)
return 0; return 0;
if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
return 0;
return 1; return 1;
} }
@ -965,9 +969,11 @@ static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
*/ */
static int static int
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff) struct anon_vma *anon_vma, struct file *file,
pgoff_t vm_pgoff,
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
{ {
if (is_mergeable_vma(vma, file, vm_flags) && if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
if (vma->vm_pgoff == vm_pgoff) if (vma->vm_pgoff == vm_pgoff)
return 1; return 1;
@ -984,9 +990,11 @@ can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
*/ */
static int static int
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff) struct anon_vma *anon_vma, struct file *file,
pgoff_t vm_pgoff,
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
{ {
if (is_mergeable_vma(vma, file, vm_flags) && if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
pgoff_t vm_pglen; pgoff_t vm_pglen;
vm_pglen = vma_pages(vma); vm_pglen = vma_pages(vma);
@ -1029,7 +1037,8 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
struct vm_area_struct *prev, unsigned long addr, struct vm_area_struct *prev, unsigned long addr,
unsigned long end, unsigned long vm_flags, unsigned long end, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file, struct anon_vma *anon_vma, struct file *file,
pgoff_t pgoff, struct mempolicy *policy) pgoff_t pgoff, struct mempolicy *policy,
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
{ {
pgoff_t pglen = (end - addr) >> PAGE_SHIFT; pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
struct vm_area_struct *area, *next; struct vm_area_struct *area, *next;
@ -1056,14 +1065,17 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
if (prev && prev->vm_end == addr && if (prev && prev->vm_end == addr &&
mpol_equal(vma_policy(prev), policy) && mpol_equal(vma_policy(prev), policy) &&
can_vma_merge_after(prev, vm_flags, can_vma_merge_after(prev, vm_flags,
anon_vma, file, pgoff)) { anon_vma, file, pgoff,
vm_userfaultfd_ctx)) {
/* /*
* OK, it can. Can we now merge in the successor as well? * OK, it can. Can we now merge in the successor as well?
*/ */
if (next && end == next->vm_start && if (next && end == next->vm_start &&
mpol_equal(policy, vma_policy(next)) && mpol_equal(policy, vma_policy(next)) &&
can_vma_merge_before(next, vm_flags, can_vma_merge_before(next, vm_flags,
anon_vma, file, pgoff+pglen) && anon_vma, file,
pgoff+pglen,
vm_userfaultfd_ctx) &&
is_mergeable_anon_vma(prev->anon_vma, is_mergeable_anon_vma(prev->anon_vma,
next->anon_vma, NULL)) { next->anon_vma, NULL)) {
/* cases 1, 6 */ /* cases 1, 6 */
@ -1084,7 +1096,8 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
if (next && end == next->vm_start && if (next && end == next->vm_start &&
mpol_equal(policy, vma_policy(next)) && mpol_equal(policy, vma_policy(next)) &&
can_vma_merge_before(next, vm_flags, can_vma_merge_before(next, vm_flags,
anon_vma, file, pgoff+pglen)) { anon_vma, file, pgoff+pglen,
vm_userfaultfd_ctx)) {
if (prev && addr < prev->vm_end) /* case 4 */ if (prev && addr < prev->vm_end) /* case 4 */
err = vma_adjust(prev, prev->vm_start, err = vma_adjust(prev, prev->vm_start,
addr, prev->vm_pgoff, NULL); addr, prev->vm_pgoff, NULL);
@ -1570,8 +1583,8 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
/* /*
* Can we just expand an old mapping? * Can we just expand an old mapping?
*/ */
vma = vma_merge(mm, prev, addr, addr + len, vm_flags, NULL, file, pgoff, vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
NULL); NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX);
if (vma) if (vma)
goto out; goto out;
@ -2757,7 +2770,7 @@ static unsigned long do_brk(unsigned long addr, unsigned long len)
/* Can we just expand an old private anonymous mapping? */ /* Can we just expand an old private anonymous mapping? */
vma = vma_merge(mm, prev, addr, addr + len, flags, vma = vma_merge(mm, prev, addr, addr + len, flags,
NULL, NULL, pgoff, NULL); NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX);
if (vma) if (vma)
goto out; goto out;
@ -2913,7 +2926,8 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
return NULL; /* should never get here */ return NULL; /* should never get here */
new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma)); vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx);
if (new_vma) { if (new_vma) {
/* /*
* Source vma may have been merged into new_vma * Source vma may have been merged into new_vma

3
mm/mprotect.c
View File

@ -292,7 +292,8 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
*/ */
pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
*pprev = vma_merge(mm, *pprev, start, end, newflags, *pprev = vma_merge(mm, *pprev, start, end, newflags,
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma)); vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx);
if (*pprev) { if (*pprev) {
vma = *pprev; vma = *pprev;
goto success; goto success;