2007-05-08 14:27:27 +08:00
|
|
|
/*
|
|
|
|
* address space "slices" (meta-segments) support
|
|
|
|
*
|
|
|
|
* Copyright (C) 2007 Benjamin Herrenschmidt, IBM Corporation.
|
|
|
|
*
|
|
|
|
* Based on hugetlb implementation
|
|
|
|
*
|
|
|
|
* Copyright (C) 2003 David Gibson, IBM Corporation.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
|
|
* (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
*/
|
|
|
|
|
|
|
|
#undef DEBUG
|
|
|
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
#include <linux/err.h>
|
|
|
|
#include <linux/spinlock.h>
|
2011-07-23 06:24:23 +08:00
|
|
|
#include <linux/export.h>
|
2014-08-20 06:55:19 +08:00
|
|
|
#include <linux/hugetlb.h>
|
2018-06-22 21:49:48 +08:00
|
|
|
#include <linux/sched/mm.h>
|
2019-02-26 12:39:35 +08:00
|
|
|
#include <linux/security.h>
|
2007-05-08 14:27:27 +08:00
|
|
|
#include <asm/mman.h>
|
|
|
|
#include <asm/mmu.h>
|
2014-10-08 16:54:52 +08:00
|
|
|
#include <asm/copro.h>
|
2014-08-20 06:55:19 +08:00
|
|
|
#include <asm/hugetlb.h>
|
2018-04-10 16:51:26 +08:00
|
|
|
#include <asm/mmu_context.h>
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2007-10-17 14:30:25 +08:00
|
|
|
static DEFINE_SPINLOCK(slice_convert_lock);
|
2007-05-08 14:27:27 +08:00
|
|
|
|
|
|
|
#ifdef DEBUG
|
|
|
|
int _slice_debug = 1;
|
|
|
|
|
2018-03-07 09:37:11 +08:00
|
|
|
static void slice_print_mask(const char *label, const struct slice_mask *mask)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
|
|
|
if (!_slice_debug)
|
|
|
|
return;
|
2018-03-07 09:37:11 +08:00
|
|
|
pr_devel("%s low_slice: %*pbl\n", label,
|
|
|
|
(int)SLICE_NUM_LOW, &mask->low_slices);
|
|
|
|
pr_devel("%s high_slice: %*pbl\n", label,
|
|
|
|
(int)SLICE_NUM_HIGH, mask->high_slices);
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
2017-03-22 11:36:52 +08:00
|
|
|
#define slice_dbg(fmt...) do { if (_slice_debug) pr_devel(fmt); } while (0)
|
2007-05-08 14:27:27 +08:00
|
|
|
|
|
|
|
#else
|
|
|
|
|
2018-03-07 09:37:11 +08:00
|
|
|
static void slice_print_mask(const char *label, const struct slice_mask *mask) {}
|
2007-05-08 14:27:27 +08:00
|
|
|
#define slice_dbg(fmt...)
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2018-08-02 17:25:55 +08:00
|
|
|
static inline bool slice_addr_is_low(unsigned long addr)
|
|
|
|
{
|
|
|
|
u64 tmp = (u64)addr;
|
|
|
|
|
|
|
|
return tmp < SLICE_LOW_TOP;
|
|
|
|
}
|
|
|
|
|
2017-03-22 11:36:48 +08:00
|
|
|
static void slice_range_to_mask(unsigned long start, unsigned long len,
|
|
|
|
struct slice_mask *ret)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
|
|
|
unsigned long end = start + len - 1;
|
2017-03-22 11:36:47 +08:00
|
|
|
|
2017-03-22 11:36:48 +08:00
|
|
|
ret->low_slices = 0;
|
2018-02-22 22:27:24 +08:00
|
|
|
if (SLICE_NUM_HIGH)
|
|
|
|
bitmap_zero(ret->high_slices, SLICE_NUM_HIGH);
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2018-08-02 17:25:55 +08:00
|
|
|
if (slice_addr_is_low(start)) {
|
2018-02-22 22:27:24 +08:00
|
|
|
unsigned long mend = min(end,
|
|
|
|
(unsigned long)(SLICE_LOW_TOP - 1));
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2017-03-22 11:36:48 +08:00
|
|
|
ret->low_slices = (1u << (GET_LOW_SLICE_INDEX(mend) + 1))
|
2017-03-22 01:29:52 +08:00
|
|
|
- (1u << GET_LOW_SLICE_INDEX(start));
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
2018-08-02 17:25:55 +08:00
|
|
|
if (SLICE_NUM_HIGH && !slice_addr_is_low(end)) {
|
2017-03-22 11:36:47 +08:00
|
|
|
unsigned long start_index = GET_HIGH_SLICE_INDEX(start);
|
|
|
|
unsigned long align_end = ALIGN(end, (1UL << SLICE_HIGH_SHIFT));
|
|
|
|
unsigned long count = GET_HIGH_SLICE_INDEX(align_end) - start_index;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2017-03-22 11:36:48 +08:00
|
|
|
bitmap_set(ret->high_slices, start_index, count);
|
2017-03-22 11:36:47 +08:00
|
|
|
}
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static int slice_area_is_free(struct mm_struct *mm, unsigned long addr,
|
|
|
|
unsigned long len)
|
|
|
|
{
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
2019-04-17 21:03:48 +08:00
|
|
|
if ((mm_ctx_slb_addr_limit(&mm->context) - len) < addr)
|
2007-05-08 14:27:27 +08:00
|
|
|
return 0;
|
|
|
|
vma = find_vma(mm, addr);
|
mm: larger stack guard gap, between vmas
Stack guard page is a useful feature to reduce a risk of stack smashing
into a different mapping. We have been using a single page gap which
is sufficient to prevent having stack adjacent to a different mapping.
But this seems to be insufficient in the light of the stack usage in
userspace. E.g. glibc uses as large as 64kB alloca() in many commonly
used functions. Others use constructs liks gid_t buffer[NGROUPS_MAX]
which is 256kB or stack strings with MAX_ARG_STRLEN.
This will become especially dangerous for suid binaries and the default
no limit for the stack size limit because those applications can be
tricked to consume a large portion of the stack and a single glibc call
could jump over the guard page. These attacks are not theoretical,
unfortunatelly.
Make those attacks less probable by increasing the stack guard gap
to 1MB (on systems with 4k pages; but make it depend on the page size
because systems with larger base pages might cap stack allocations in
the PAGE_SIZE units) which should cover larger alloca() and VLA stack
allocations. It is obviously not a full fix because the problem is
somehow inherent, but it should reduce attack space a lot.
One could argue that the gap size should be configurable from userspace,
but that can be done later when somebody finds that the new 1MB is wrong
for some special case applications. For now, add a kernel command line
option (stack_guard_gap) to specify the stack gap size (in page units).
Implementation wise, first delete all the old code for stack guard page:
because although we could get away with accounting one extra page in a
stack vma, accounting a larger gap can break userspace - case in point,
a program run with "ulimit -S -v 20000" failed when the 1MB gap was
counted for RLIMIT_AS; similar problems could come with RLIMIT_MLOCK
and strict non-overcommit mode.
Instead of keeping gap inside the stack vma, maintain the stack guard
gap as a gap between vmas: using vm_start_gap() in place of vm_start
(or vm_end_gap() in place of vm_end if VM_GROWSUP) in just those few
places which need to respect the gap - mainly arch_get_unmapped_area(),
and and the vma tree's subtree_gap support for that.
Original-patch-by: Oleg Nesterov <oleg@redhat.com>
Original-patch-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Hugh Dickins <hughd@google.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Tested-by: Helge Deller <deller@gmx.de> # parisc
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-06-19 19:03:24 +08:00
|
|
|
return (!vma || (addr + len) <= vm_start_gap(vma));
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static int slice_low_has_vma(struct mm_struct *mm, unsigned long slice)
|
|
|
|
{
|
|
|
|
return !slice_area_is_free(mm, slice << SLICE_LOW_SHIFT,
|
|
|
|
1ul << SLICE_LOW_SHIFT);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int slice_high_has_vma(struct mm_struct *mm, unsigned long slice)
|
|
|
|
{
|
|
|
|
unsigned long start = slice << SLICE_HIGH_SHIFT;
|
|
|
|
unsigned long end = start + (1ul << SLICE_HIGH_SHIFT);
|
|
|
|
|
2018-02-22 22:27:24 +08:00
|
|
|
#ifdef CONFIG_PPC64
|
2007-05-08 14:27:27 +08:00
|
|
|
/* Hack, so that each addresses is controlled by exactly one
|
|
|
|
* of the high or low area bitmaps, the first high area starts
|
|
|
|
* at 4GB, not 0 */
|
|
|
|
if (start == 0)
|
|
|
|
start = SLICE_LOW_TOP;
|
2018-02-22 22:27:24 +08:00
|
|
|
#endif
|
2007-05-08 14:27:27 +08:00
|
|
|
|
|
|
|
return !slice_area_is_free(mm, start, end - start);
|
|
|
|
}
|
|
|
|
|
2017-11-10 12:55:07 +08:00
|
|
|
static void slice_mask_for_free(struct mm_struct *mm, struct slice_mask *ret,
|
|
|
|
unsigned long high_limit)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
|
|
|
unsigned long i;
|
|
|
|
|
2017-03-22 11:36:48 +08:00
|
|
|
ret->low_slices = 0;
|
2018-02-22 22:27:24 +08:00
|
|
|
if (SLICE_NUM_HIGH)
|
|
|
|
bitmap_zero(ret->high_slices, SLICE_NUM_HIGH);
|
2017-03-22 11:36:47 +08:00
|
|
|
|
2007-05-08 14:27:27 +08:00
|
|
|
for (i = 0; i < SLICE_NUM_LOW; i++)
|
|
|
|
if (!slice_low_has_vma(mm, i))
|
2017-03-22 11:36:48 +08:00
|
|
|
ret->low_slices |= 1u << i;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2018-08-02 17:25:55 +08:00
|
|
|
if (slice_addr_is_low(high_limit - 1))
|
2017-03-22 11:36:48 +08:00
|
|
|
return;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2017-11-10 12:55:07 +08:00
|
|
|
for (i = 0; i < GET_HIGH_SLICE_INDEX(high_limit); i++)
|
2007-05-08 14:27:27 +08:00
|
|
|
if (!slice_high_has_vma(mm, i))
|
2017-03-22 11:36:48 +08:00
|
|
|
__set_bit(i, ret->high_slices);
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
2018-03-07 09:37:13 +08:00
|
|
|
static bool slice_check_range_fits(struct mm_struct *mm,
|
|
|
|
const struct slice_mask *available,
|
|
|
|
unsigned long start, unsigned long len)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
2018-03-07 09:37:13 +08:00
|
|
|
unsigned long end = start + len - 1;
|
|
|
|
u64 low_slices = 0;
|
2017-03-22 11:36:47 +08:00
|
|
|
|
2018-08-02 17:25:55 +08:00
|
|
|
if (slice_addr_is_low(start)) {
|
2018-03-07 09:37:13 +08:00
|
|
|
unsigned long mend = min(end,
|
|
|
|
(unsigned long)(SLICE_LOW_TOP - 1));
|
2018-02-22 22:27:24 +08:00
|
|
|
|
2018-03-07 09:37:13 +08:00
|
|
|
low_slices = (1u << (GET_LOW_SLICE_INDEX(mend) + 1))
|
|
|
|
- (1u << GET_LOW_SLICE_INDEX(start));
|
|
|
|
}
|
|
|
|
if ((low_slices & available->low_slices) != low_slices)
|
|
|
|
return false;
|
2017-03-22 11:36:47 +08:00
|
|
|
|
2018-08-02 17:25:55 +08:00
|
|
|
if (SLICE_NUM_HIGH && !slice_addr_is_low(end)) {
|
2018-03-07 09:37:13 +08:00
|
|
|
unsigned long start_index = GET_HIGH_SLICE_INDEX(start);
|
|
|
|
unsigned long align_end = ALIGN(end, (1UL << SLICE_HIGH_SHIFT));
|
|
|
|
unsigned long count = GET_HIGH_SLICE_INDEX(align_end) - start_index;
|
|
|
|
unsigned long i;
|
|
|
|
|
|
|
|
for (i = start_index; i < start_index + count; i++) {
|
|
|
|
if (!test_bit(i, available->high_slices))
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
2018-10-02 21:56:39 +08:00
|
|
|
static void slice_flush_segments(void *parm)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_PPC64
|
|
|
|
struct mm_struct *mm = parm;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
if (mm != current->active_mm)
|
|
|
|
return;
|
|
|
|
|
|
|
|
copy_mm_to_paca(current->active_mm);
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
2018-10-02 22:27:58 +08:00
|
|
|
slb_flush_and_restore_bolted();
|
2018-10-02 21:56:39 +08:00
|
|
|
local_irq_restore(flags);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2018-03-07 09:37:11 +08:00
|
|
|
static void slice_convert(struct mm_struct *mm,
|
|
|
|
const struct slice_mask *mask, int psize)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
2012-09-10 10:52:52 +08:00
|
|
|
int index, mask_index;
|
2007-05-08 14:27:27 +08:00
|
|
|
/* Write the new slice psize bits */
|
2018-02-22 22:27:28 +08:00
|
|
|
unsigned char *hpsizes, *lpsizes;
|
2018-03-07 09:37:12 +08:00
|
|
|
struct slice_mask *psize_mask, *old_mask;
|
2007-05-08 14:27:27 +08:00
|
|
|
unsigned long i, flags;
|
2018-03-07 09:37:12 +08:00
|
|
|
int old_psize;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
|
|
|
slice_dbg("slice_convert(mm=%p, psize=%d)\n", mm, psize);
|
|
|
|
slice_print_mask(" mask", mask);
|
|
|
|
|
2019-04-25 22:29:29 +08:00
|
|
|
psize_mask = slice_mask_for_size(&mm->context, psize);
|
2018-03-07 09:37:12 +08:00
|
|
|
|
2007-05-08 14:27:27 +08:00
|
|
|
/* We need to use a spinlock here to protect against
|
|
|
|
* concurrent 64k -> 4k demotion ...
|
|
|
|
*/
|
|
|
|
spin_lock_irqsave(&slice_convert_lock, flags);
|
|
|
|
|
2019-04-17 21:03:48 +08:00
|
|
|
lpsizes = mm_ctx_low_slices(&mm->context);
|
2018-03-07 09:37:10 +08:00
|
|
|
for (i = 0; i < SLICE_NUM_LOW; i++) {
|
2018-03-07 09:37:11 +08:00
|
|
|
if (!(mask->low_slices & (1u << i)))
|
2018-03-07 09:37:10 +08:00
|
|
|
continue;
|
|
|
|
|
|
|
|
mask_index = i & 0x1;
|
|
|
|
index = i >> 1;
|
2018-03-07 09:37:12 +08:00
|
|
|
|
|
|
|
/* Update the slice_mask */
|
|
|
|
old_psize = (lpsizes[index] >> (mask_index * 4)) & 0xf;
|
2019-04-25 22:29:29 +08:00
|
|
|
old_mask = slice_mask_for_size(&mm->context, old_psize);
|
2018-03-07 09:37:12 +08:00
|
|
|
old_mask->low_slices &= ~(1u << i);
|
|
|
|
psize_mask->low_slices |= 1u << i;
|
|
|
|
|
|
|
|
/* Update the sizes array */
|
2018-03-07 09:37:10 +08:00
|
|
|
lpsizes[index] = (lpsizes[index] & ~(0xf << (mask_index * 4))) |
|
2018-02-22 22:27:28 +08:00
|
|
|
(((unsigned long)psize) << (mask_index * 4));
|
2018-03-07 09:37:10 +08:00
|
|
|
}
|
2012-09-10 10:52:52 +08:00
|
|
|
|
2019-04-17 21:03:48 +08:00
|
|
|
hpsizes = mm_ctx_high_slices(&mm->context);
|
|
|
|
for (i = 0; i < GET_HIGH_SLICE_INDEX(mm_ctx_slb_addr_limit(&mm->context)); i++) {
|
2018-03-07 09:37:11 +08:00
|
|
|
if (!test_bit(i, mask->high_slices))
|
2018-03-07 09:37:10 +08:00
|
|
|
continue;
|
|
|
|
|
2012-09-10 10:52:52 +08:00
|
|
|
mask_index = i & 0x1;
|
|
|
|
index = i >> 1;
|
2018-03-07 09:37:12 +08:00
|
|
|
|
|
|
|
/* Update the slice_mask */
|
|
|
|
old_psize = (hpsizes[index] >> (mask_index * 4)) & 0xf;
|
2019-04-25 22:29:29 +08:00
|
|
|
old_mask = slice_mask_for_size(&mm->context, old_psize);
|
2018-03-07 09:37:12 +08:00
|
|
|
__clear_bit(i, old_mask->high_slices);
|
|
|
|
__set_bit(i, psize_mask->high_slices);
|
|
|
|
|
|
|
|
/* Update the sizes array */
|
2018-03-07 09:37:10 +08:00
|
|
|
hpsizes[index] = (hpsizes[index] & ~(0xf << (mask_index * 4))) |
|
2012-09-10 10:52:52 +08:00
|
|
|
(((unsigned long)psize) << (mask_index * 4));
|
|
|
|
}
|
2007-05-08 14:27:27 +08:00
|
|
|
|
|
|
|
slice_dbg(" lsps=%lx, hsps=%lx\n",
|
2019-04-17 21:03:48 +08:00
|
|
|
(unsigned long)mm_ctx_low_slices(&mm->context),
|
|
|
|
(unsigned long)mm_ctx_high_slices(&mm->context));
|
2007-05-08 14:27:27 +08:00
|
|
|
|
|
|
|
spin_unlock_irqrestore(&slice_convert_lock, flags);
|
|
|
|
|
2014-10-08 16:54:52 +08:00
|
|
|
copro_flush_all_slbs(mm);
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
2013-04-30 02:53:53 +08:00
|
|
|
/*
|
|
|
|
* Compute which slice addr is part of;
|
|
|
|
* set *boundary_addr to the start or end boundary of that slice
|
|
|
|
* (depending on 'end' parameter);
|
|
|
|
* return boolean indicating if the slice is marked as available in the
|
|
|
|
* 'available' slice_mark.
|
|
|
|
*/
|
|
|
|
static bool slice_scan_available(unsigned long addr,
|
2018-03-07 09:37:11 +08:00
|
|
|
const struct slice_mask *available,
|
|
|
|
int end, unsigned long *boundary_addr)
|
2013-04-30 02:53:53 +08:00
|
|
|
{
|
|
|
|
unsigned long slice;
|
2018-08-02 17:25:55 +08:00
|
|
|
if (slice_addr_is_low(addr)) {
|
2013-04-30 02:53:53 +08:00
|
|
|
slice = GET_LOW_SLICE_INDEX(addr);
|
|
|
|
*boundary_addr = (slice + end) << SLICE_LOW_SHIFT;
|
2018-03-07 09:37:11 +08:00
|
|
|
return !!(available->low_slices & (1u << slice));
|
2013-04-30 02:53:53 +08:00
|
|
|
} else {
|
|
|
|
slice = GET_HIGH_SLICE_INDEX(addr);
|
|
|
|
*boundary_addr = (slice + end) ?
|
|
|
|
((slice + end) << SLICE_HIGH_SHIFT) : SLICE_LOW_TOP;
|
2018-03-07 09:37:11 +08:00
|
|
|
return !!test_bit(slice, available->high_slices);
|
2013-04-30 02:53:53 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-05-08 14:27:27 +08:00
|
|
|
static unsigned long slice_find_area_bottomup(struct mm_struct *mm,
|
|
|
|
unsigned long len,
|
2018-03-07 09:37:11 +08:00
|
|
|
const struct slice_mask *available,
|
2017-03-30 19:05:21 +08:00
|
|
|
int psize, unsigned long high_limit)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
|
|
|
int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
|
2013-04-30 02:53:53 +08:00
|
|
|
unsigned long addr, found, next_end;
|
|
|
|
struct vm_unmapped_area_info info;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2013-04-30 02:53:53 +08:00
|
|
|
info.flags = 0;
|
|
|
|
info.length = len;
|
|
|
|
info.align_mask = PAGE_MASK & ((1ul << pshift) - 1);
|
|
|
|
info.align_offset = 0;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2013-04-30 02:53:53 +08:00
|
|
|
addr = TASK_UNMAPPED_BASE;
|
2017-03-30 19:05:21 +08:00
|
|
|
/*
|
|
|
|
* Check till the allow max value for this mmap request
|
|
|
|
*/
|
|
|
|
while (addr < high_limit) {
|
2013-04-30 02:53:53 +08:00
|
|
|
info.low_limit = addr;
|
|
|
|
if (!slice_scan_available(addr, available, 1, &addr))
|
2007-05-08 14:27:27 +08:00
|
|
|
continue;
|
2013-04-30 02:53:53 +08:00
|
|
|
|
|
|
|
next_slice:
|
|
|
|
/*
|
|
|
|
* At this point [info.low_limit; addr) covers
|
|
|
|
* available slices only and ends at a slice boundary.
|
|
|
|
* Check if we need to reduce the range, or if we can
|
|
|
|
* extend it to cover the next available slice.
|
|
|
|
*/
|
2017-04-14 03:18:21 +08:00
|
|
|
if (addr >= high_limit)
|
|
|
|
addr = high_limit;
|
2013-04-30 02:53:53 +08:00
|
|
|
else if (slice_scan_available(addr, available, 1, &next_end)) {
|
|
|
|
addr = next_end;
|
|
|
|
goto next_slice;
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
2013-04-30 02:53:53 +08:00
|
|
|
info.high_limit = addr;
|
|
|
|
|
|
|
|
found = vm_unmapped_area(&info);
|
|
|
|
if (!(found & ~PAGE_MASK))
|
|
|
|
return found;
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long slice_find_area_topdown(struct mm_struct *mm,
|
|
|
|
unsigned long len,
|
2018-03-07 09:37:11 +08:00
|
|
|
const struct slice_mask *available,
|
2017-03-30 19:05:21 +08:00
|
|
|
int psize, unsigned long high_limit)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
|
|
|
int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
|
2013-04-30 02:53:53 +08:00
|
|
|
unsigned long addr, found, prev;
|
|
|
|
struct vm_unmapped_area_info info;
|
2019-02-26 12:39:35 +08:00
|
|
|
unsigned long min_addr = max(PAGE_SIZE, mmap_min_addr);
|
2013-04-30 02:53:53 +08:00
|
|
|
|
|
|
|
info.flags = VM_UNMAPPED_AREA_TOPDOWN;
|
|
|
|
info.length = len;
|
|
|
|
info.align_mask = PAGE_MASK & ((1ul << pshift) - 1);
|
|
|
|
info.align_offset = 0;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
|
|
|
addr = mm->mmap_base;
|
2017-03-30 19:05:21 +08:00
|
|
|
/*
|
|
|
|
* If we are trying to allocate above DEFAULT_MAP_WINDOW
|
|
|
|
* Add the different to the mmap_base.
|
|
|
|
* Only for that request for which high_limit is above
|
|
|
|
* DEFAULT_MAP_WINDOW we should apply this.
|
|
|
|
*/
|
2017-11-10 01:27:40 +08:00
|
|
|
if (high_limit > DEFAULT_MAP_WINDOW)
|
2019-04-17 21:03:48 +08:00
|
|
|
addr += mm_ctx_slb_addr_limit(&mm->context) - DEFAULT_MAP_WINDOW;
|
2017-03-30 19:05:21 +08:00
|
|
|
|
2019-02-26 12:39:35 +08:00
|
|
|
while (addr > min_addr) {
|
2013-04-30 02:53:53 +08:00
|
|
|
info.high_limit = addr;
|
|
|
|
if (!slice_scan_available(addr - 1, available, 0, &addr))
|
2007-05-08 14:27:27 +08:00
|
|
|
continue;
|
|
|
|
|
2013-04-30 02:53:53 +08:00
|
|
|
prev_slice:
|
2007-05-08 14:27:27 +08:00
|
|
|
/*
|
2013-04-30 02:53:53 +08:00
|
|
|
* At this point [addr; info.high_limit) covers
|
|
|
|
* available slices only and starts at a slice boundary.
|
|
|
|
* Check if we need to reduce the range, or if we can
|
|
|
|
* extend it to cover the previous available slice.
|
2007-05-08 14:27:27 +08:00
|
|
|
*/
|
2019-02-26 12:39:35 +08:00
|
|
|
if (addr < min_addr)
|
|
|
|
addr = min_addr;
|
2013-04-30 02:53:53 +08:00
|
|
|
else if (slice_scan_available(addr - 1, available, 0, &prev)) {
|
|
|
|
addr = prev;
|
|
|
|
goto prev_slice;
|
|
|
|
}
|
|
|
|
info.low_limit = addr;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2013-04-30 02:53:53 +08:00
|
|
|
found = vm_unmapped_area(&info);
|
|
|
|
if (!(found & ~PAGE_MASK))
|
|
|
|
return found;
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A failed mmap() very likely causes application failure,
|
|
|
|
* so fall back to the bottom-up function here. This scenario
|
|
|
|
* can happen with large stack limits and large mmap()
|
|
|
|
* allocations.
|
|
|
|
*/
|
2017-03-30 19:05:21 +08:00
|
|
|
return slice_find_area_bottomup(mm, len, available, psize, high_limit);
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,
|
2018-03-07 09:37:11 +08:00
|
|
|
const struct slice_mask *mask, int psize,
|
2017-03-30 19:05:21 +08:00
|
|
|
int topdown, unsigned long high_limit)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
|
|
|
if (topdown)
|
2017-03-30 19:05:21 +08:00
|
|
|
return slice_find_area_topdown(mm, len, mask, psize, high_limit);
|
2007-05-08 14:27:27 +08:00
|
|
|
else
|
2017-03-30 19:05:21 +08:00
|
|
|
return slice_find_area_bottomup(mm, len, mask, psize, high_limit);
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
2018-03-07 09:37:14 +08:00
|
|
|
static inline void slice_copy_mask(struct slice_mask *dst,
|
2018-03-07 09:37:11 +08:00
|
|
|
const struct slice_mask *src)
|
2017-03-22 11:36:47 +08:00
|
|
|
{
|
2018-03-07 09:37:14 +08:00
|
|
|
dst->low_slices = src->low_slices;
|
2018-02-22 22:27:24 +08:00
|
|
|
if (!SLICE_NUM_HIGH)
|
|
|
|
return;
|
2018-03-07 09:37:14 +08:00
|
|
|
bitmap_copy(dst->high_slices, src->high_slices, SLICE_NUM_HIGH);
|
2017-03-22 11:36:47 +08:00
|
|
|
}
|
|
|
|
|
2018-03-07 09:37:14 +08:00
|
|
|
static inline void slice_or_mask(struct slice_mask *dst,
|
|
|
|
const struct slice_mask *src1,
|
|
|
|
const struct slice_mask *src2)
|
2017-03-22 11:36:47 +08:00
|
|
|
{
|
2018-03-07 09:37:14 +08:00
|
|
|
dst->low_slices = src1->low_slices | src2->low_slices;
|
|
|
|
if (!SLICE_NUM_HIGH)
|
|
|
|
return;
|
|
|
|
bitmap_or(dst->high_slices, src1->high_slices, src2->high_slices, SLICE_NUM_HIGH);
|
|
|
|
}
|
2017-03-22 11:36:47 +08:00
|
|
|
|
2018-03-07 09:37:14 +08:00
|
|
|
static inline void slice_andnot_mask(struct slice_mask *dst,
|
|
|
|
const struct slice_mask *src1,
|
|
|
|
const struct slice_mask *src2)
|
|
|
|
{
|
|
|
|
dst->low_slices = src1->low_slices & ~src2->low_slices;
|
2018-02-22 22:27:24 +08:00
|
|
|
if (!SLICE_NUM_HIGH)
|
|
|
|
return;
|
2018-03-07 09:37:14 +08:00
|
|
|
bitmap_andnot(dst->high_slices, src1->high_slices, src2->high_slices, SLICE_NUM_HIGH);
|
2017-03-22 11:36:47 +08:00
|
|
|
}
|
2008-06-18 13:29:12 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_PPC_64K_PAGES
|
|
|
|
#define MMU_PAGE_BASE MMU_PAGE_64K
|
|
|
|
#else
|
|
|
|
#define MMU_PAGE_BASE MMU_PAGE_4K
|
|
|
|
#endif
|
|
|
|
|
2007-05-08 14:27:27 +08:00
|
|
|
unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
|
|
|
|
unsigned long flags, unsigned int psize,
|
2013-04-30 02:53:52 +08:00
|
|
|
int topdown)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
|
|
|
struct slice_mask good_mask;
|
2017-03-22 11:36:47 +08:00
|
|
|
struct slice_mask potential_mask;
|
2018-03-07 09:37:16 +08:00
|
|
|
const struct slice_mask *maskp;
|
|
|
|
const struct slice_mask *compat_maskp = NULL;
|
2007-05-08 14:27:27 +08:00
|
|
|
int fixed = (flags & MAP_FIXED);
|
|
|
|
int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
|
2017-11-10 01:27:36 +08:00
|
|
|
unsigned long page_size = 1UL << pshift;
|
2007-05-08 14:27:27 +08:00
|
|
|
struct mm_struct *mm = current->mm;
|
2008-06-18 13:29:12 +08:00
|
|
|
unsigned long newaddr;
|
2017-03-30 19:05:21 +08:00
|
|
|
unsigned long high_limit;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2017-11-10 01:27:36 +08:00
|
|
|
high_limit = DEFAULT_MAP_WINDOW;
|
2017-11-10 01:27:38 +08:00
|
|
|
if (addr >= high_limit || (fixed && (addr + len > high_limit)))
|
2017-11-10 01:27:36 +08:00
|
|
|
high_limit = TASK_SIZE;
|
|
|
|
|
|
|
|
if (len > high_limit)
|
|
|
|
return -ENOMEM;
|
|
|
|
if (len & (page_size - 1))
|
|
|
|
return -EINVAL;
|
|
|
|
if (fixed) {
|
|
|
|
if (addr & (page_size - 1))
|
|
|
|
return -EINVAL;
|
|
|
|
if (addr > high_limit - len)
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2019-04-17 21:03:48 +08:00
|
|
|
if (high_limit > mm_ctx_slb_addr_limit(&mm->context)) {
|
2018-03-07 09:37:12 +08:00
|
|
|
/*
|
|
|
|
* Increasing the slb_addr_limit does not require
|
|
|
|
* slice mask cache to be recalculated because it should
|
|
|
|
* be already initialised beyond the old address limit.
|
|
|
|
*/
|
2019-04-17 21:03:48 +08:00
|
|
|
mm_ctx_set_slb_addr_limit(&mm->context, high_limit);
|
2018-10-02 21:56:39 +08:00
|
|
|
|
|
|
|
on_each_cpu(slice_flush_segments, mm, 1);
|
2017-03-30 19:05:21 +08:00
|
|
|
}
|
2017-11-10 01:27:36 +08:00
|
|
|
|
2007-05-08 14:27:27 +08:00
|
|
|
/* Sanity checks */
|
|
|
|
BUG_ON(mm->task_size == 0);
|
2019-04-17 21:03:48 +08:00
|
|
|
BUG_ON(mm_ctx_slb_addr_limit(&mm->context) == 0);
|
2016-04-29 21:26:09 +08:00
|
|
|
VM_BUG_ON(radix_enabled());
|
2007-05-08 14:27:27 +08:00
|
|
|
|
|
|
|
slice_dbg("slice_get_unmapped_area(mm=%p, psize=%d...\n", mm, psize);
|
2013-04-30 02:53:52 +08:00
|
|
|
slice_dbg(" addr=%lx, len=%lx, flags=%lx, topdown=%d\n",
|
|
|
|
addr, len, flags, topdown);
|
2007-05-08 14:27:27 +08:00
|
|
|
|
|
|
|
/* If hint, make sure it matches our alignment restrictions */
|
|
|
|
if (!fixed && addr) {
|
2017-11-10 01:27:36 +08:00
|
|
|
addr = _ALIGN_UP(addr, page_size);
|
2007-05-08 14:27:27 +08:00
|
|
|
slice_dbg(" aligned addr=%lx\n", addr);
|
2008-06-18 13:29:12 +08:00
|
|
|
/* Ignore hint if it's too large or overlaps a VMA */
|
2019-02-26 12:39:35 +08:00
|
|
|
if (addr > high_limit - len || addr < mmap_min_addr ||
|
2008-06-18 13:29:12 +08:00
|
|
|
!slice_area_is_free(mm, addr, len))
|
|
|
|
addr = 0;
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
2008-06-18 13:29:12 +08:00
|
|
|
/* First make up a "good" mask of slices that have the right size
|
2007-05-08 14:27:27 +08:00
|
|
|
* already
|
|
|
|
*/
|
2019-04-25 22:29:29 +08:00
|
|
|
maskp = slice_mask_for_size(&mm->context, psize);
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2008-06-18 13:29:12 +08:00
|
|
|
/*
|
|
|
|
* Here "good" means slices that are already the right page size,
|
|
|
|
* "compat" means slices that have a compatible page size (i.e.
|
|
|
|
* 4k in a 64k pagesize kernel), and "free" means slices without
|
|
|
|
* any VMAs.
|
|
|
|
*
|
|
|
|
* If MAP_FIXED:
|
|
|
|
* check if fits in good | compat => OK
|
|
|
|
* check if fits in good | compat | free => convert free
|
|
|
|
* else bad
|
|
|
|
* If have hint:
|
|
|
|
* check if hint fits in good => OK
|
|
|
|
* check if hint fits in good | free => convert free
|
|
|
|
* Otherwise:
|
|
|
|
* search in good, found => OK
|
|
|
|
* search in good | free, found => convert free
|
|
|
|
* search in good | compat | free, found => convert free.
|
|
|
|
*/
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2018-03-07 09:37:16 +08:00
|
|
|
/*
|
|
|
|
* If we support combo pages, we can allow 64k pages in 4k slices
|
|
|
|
* The mask copies could be avoided in most cases here if we had
|
|
|
|
* a pointer to good mask for the next code to use.
|
|
|
|
*/
|
|
|
|
if (IS_ENABLED(CONFIG_PPC_64K_PAGES) && psize == MMU_PAGE_64K) {
|
2019-04-25 22:29:29 +08:00
|
|
|
compat_maskp = slice_mask_for_size(&mm->context, MMU_PAGE_4K);
|
2008-06-18 13:29:12 +08:00
|
|
|
if (fixed)
|
2018-03-07 09:37:16 +08:00
|
|
|
slice_or_mask(&good_mask, maskp, compat_maskp);
|
|
|
|
else
|
|
|
|
slice_copy_mask(&good_mask, maskp);
|
|
|
|
} else {
|
|
|
|
slice_copy_mask(&good_mask, maskp);
|
2008-06-18 13:29:12 +08:00
|
|
|
}
|
2018-03-07 09:37:16 +08:00
|
|
|
|
|
|
|
slice_print_mask(" good_mask", &good_mask);
|
|
|
|
if (compat_maskp)
|
|
|
|
slice_print_mask(" compat_mask", compat_maskp);
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2008-06-18 13:29:12 +08:00
|
|
|
/* First check hint if it's valid or if we have MAP_FIXED */
|
|
|
|
if (addr != 0 || fixed) {
|
2007-05-08 14:27:27 +08:00
|
|
|
/* Check if we fit in the good mask. If we do, we just return,
|
|
|
|
* nothing else to do
|
|
|
|
*/
|
2018-03-07 09:37:13 +08:00
|
|
|
if (slice_check_range_fits(mm, &good_mask, addr, len)) {
|
2007-05-08 14:27:27 +08:00
|
|
|
slice_dbg(" fits good !\n");
|
2018-03-26 18:04:47 +08:00
|
|
|
newaddr = addr;
|
|
|
|
goto return_addr;
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
2008-06-18 13:29:12 +08:00
|
|
|
} else {
|
|
|
|
/* Now let's see if we can find something in the existing
|
|
|
|
* slices for that size
|
2007-05-08 14:27:27 +08:00
|
|
|
*/
|
2018-03-07 09:37:11 +08:00
|
|
|
newaddr = slice_find_area(mm, len, &good_mask,
|
2017-03-30 19:05:21 +08:00
|
|
|
psize, topdown, high_limit);
|
2008-06-18 13:29:12 +08:00
|
|
|
if (newaddr != -ENOMEM) {
|
|
|
|
/* Found within the good mask, we don't have to setup,
|
|
|
|
* we thus return directly
|
|
|
|
*/
|
|
|
|
slice_dbg(" found area at 0x%lx\n", newaddr);
|
2018-03-26 18:04:47 +08:00
|
|
|
goto return_addr;
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
}
|
2017-11-10 12:55:07 +08:00
|
|
|
/*
|
|
|
|
* We don't fit in the good mask, check what other slices are
|
2008-06-18 13:29:12 +08:00
|
|
|
* empty and thus can be converted
|
|
|
|
*/
|
2017-11-10 12:55:07 +08:00
|
|
|
slice_mask_for_free(mm, &potential_mask, high_limit);
|
2018-03-07 09:37:14 +08:00
|
|
|
slice_or_mask(&potential_mask, &potential_mask, &good_mask);
|
2018-03-07 09:37:11 +08:00
|
|
|
slice_print_mask(" potential", &potential_mask);
|
2008-06-18 13:29:12 +08:00
|
|
|
|
2018-03-07 09:37:13 +08:00
|
|
|
if (addr != 0 || fixed) {
|
|
|
|
if (slice_check_range_fits(mm, &potential_mask, addr, len)) {
|
|
|
|
slice_dbg(" fits potential !\n");
|
2018-03-26 18:04:47 +08:00
|
|
|
newaddr = addr;
|
2018-03-07 09:37:13 +08:00
|
|
|
goto convert;
|
|
|
|
}
|
2008-06-18 13:29:12 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* If we have MAP_FIXED and failed the above steps, then error out */
|
2007-05-08 14:27:27 +08:00
|
|
|
if (fixed)
|
|
|
|
return -EBUSY;
|
|
|
|
|
|
|
|
slice_dbg(" search...\n");
|
|
|
|
|
2008-06-18 13:29:12 +08:00
|
|
|
/* If we had a hint that didn't work out, see if we can fit
|
|
|
|
* anywhere in the good area.
|
2007-05-08 14:27:27 +08:00
|
|
|
*/
|
2008-06-18 13:29:12 +08:00
|
|
|
if (addr) {
|
2018-03-26 18:04:47 +08:00
|
|
|
newaddr = slice_find_area(mm, len, &good_mask,
|
|
|
|
psize, topdown, high_limit);
|
|
|
|
if (newaddr != -ENOMEM) {
|
|
|
|
slice_dbg(" found area at 0x%lx\n", newaddr);
|
|
|
|
goto return_addr;
|
2008-06-18 13:29:12 +08:00
|
|
|
}
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Now let's see if we can find something in the existing slices
|
2008-06-18 13:29:12 +08:00
|
|
|
* for that size plus free slices
|
2007-05-08 14:27:27 +08:00
|
|
|
*/
|
2018-03-26 18:04:47 +08:00
|
|
|
newaddr = slice_find_area(mm, len, &potential_mask,
|
|
|
|
psize, topdown, high_limit);
|
2008-06-18 13:29:12 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_PPC_64K_PAGES
|
2018-03-26 18:04:47 +08:00
|
|
|
if (newaddr == -ENOMEM && psize == MMU_PAGE_64K) {
|
2008-06-18 13:29:12 +08:00
|
|
|
/* retry the search with 4k-page slices included */
|
2018-03-07 09:37:16 +08:00
|
|
|
slice_or_mask(&potential_mask, &potential_mask, compat_maskp);
|
2018-03-26 18:04:47 +08:00
|
|
|
newaddr = slice_find_area(mm, len, &potential_mask,
|
|
|
|
psize, topdown, high_limit);
|
2008-06-18 13:29:12 +08:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2018-03-26 18:04:47 +08:00
|
|
|
if (newaddr == -ENOMEM)
|
2007-05-08 14:27:27 +08:00
|
|
|
return -ENOMEM;
|
|
|
|
|
2018-03-26 18:04:47 +08:00
|
|
|
slice_range_to_mask(newaddr, len, &potential_mask);
|
|
|
|
slice_dbg(" found potential area at 0x%lx\n", newaddr);
|
2018-03-07 09:37:16 +08:00
|
|
|
slice_print_mask(" mask", &potential_mask);
|
2007-05-08 14:27:27 +08:00
|
|
|
|
|
|
|
convert:
|
2018-03-26 18:04:48 +08:00
|
|
|
/*
|
|
|
|
* Try to allocate the context before we do slice convert
|
|
|
|
* so that we handle the context allocation failure gracefully.
|
|
|
|
*/
|
|
|
|
if (need_extra_context(mm, newaddr)) {
|
|
|
|
if (alloc_extended_context(mm, newaddr) < 0)
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2018-03-07 09:37:16 +08:00
|
|
|
slice_andnot_mask(&potential_mask, &potential_mask, &good_mask);
|
|
|
|
if (compat_maskp && !fixed)
|
|
|
|
slice_andnot_mask(&potential_mask, &potential_mask, compat_maskp);
|
|
|
|
if (potential_mask.low_slices ||
|
|
|
|
(SLICE_NUM_HIGH &&
|
|
|
|
!bitmap_empty(potential_mask.high_slices, SLICE_NUM_HIGH))) {
|
|
|
|
slice_convert(mm, &potential_mask, psize);
|
2008-06-18 13:29:12 +08:00
|
|
|
if (psize > MMU_PAGE_BASE)
|
2018-10-02 21:56:39 +08:00
|
|
|
on_each_cpu(slice_flush_segments, mm, 1);
|
2008-06-18 13:29:12 +08:00
|
|
|
}
|
2018-03-26 18:04:48 +08:00
|
|
|
return newaddr;
|
2018-03-26 18:04:47 +08:00
|
|
|
|
|
|
|
return_addr:
|
2018-03-26 18:04:48 +08:00
|
|
|
if (need_extra_context(mm, newaddr)) {
|
|
|
|
if (alloc_extended_context(mm, newaddr) < 0)
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
2018-03-26 18:04:47 +08:00
|
|
|
return newaddr;
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(slice_get_unmapped_area);
|
|
|
|
|
|
|
|
unsigned long arch_get_unmapped_area(struct file *filp,
|
|
|
|
unsigned long addr,
|
|
|
|
unsigned long len,
|
|
|
|
unsigned long pgoff,
|
|
|
|
unsigned long flags)
|
|
|
|
{
|
|
|
|
return slice_get_unmapped_area(addr, len, flags,
|
2019-04-17 21:03:48 +08:00
|
|
|
mm_ctx_user_psize(¤t->mm->context), 0);
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long arch_get_unmapped_area_topdown(struct file *filp,
|
|
|
|
const unsigned long addr0,
|
|
|
|
const unsigned long len,
|
|
|
|
const unsigned long pgoff,
|
|
|
|
const unsigned long flags)
|
|
|
|
{
|
|
|
|
return slice_get_unmapped_area(addr0, len, flags,
|
2019-04-17 21:03:48 +08:00
|
|
|
mm_ctx_user_psize(¤t->mm->context), 1);
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr)
|
|
|
|
{
|
2018-02-22 22:27:28 +08:00
|
|
|
unsigned char *psizes;
|
2012-09-10 10:52:52 +08:00
|
|
|
int index, mask_index;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2018-03-07 09:37:17 +08:00
|
|
|
VM_BUG_ON(radix_enabled());
|
|
|
|
|
2018-08-02 17:25:55 +08:00
|
|
|
if (slice_addr_is_low(addr)) {
|
2019-04-17 21:03:48 +08:00
|
|
|
psizes = mm_ctx_low_slices(&mm->context);
|
2007-05-08 14:27:27 +08:00
|
|
|
index = GET_LOW_SLICE_INDEX(addr);
|
2018-02-22 22:27:28 +08:00
|
|
|
} else {
|
2019-04-17 21:03:48 +08:00
|
|
|
psizes = mm_ctx_high_slices(&mm->context);
|
2018-02-22 22:27:28 +08:00
|
|
|
index = GET_HIGH_SLICE_INDEX(addr);
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
2012-09-10 10:52:52 +08:00
|
|
|
mask_index = index & 0x1;
|
2018-02-22 22:27:28 +08:00
|
|
|
return (psizes[index >> 1] >> (mask_index * 4)) & 0xf;
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(get_slice_psize);
|
|
|
|
|
2018-03-07 09:37:09 +08:00
|
|
|
void slice_init_new_context_exec(struct mm_struct *mm)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
2018-02-22 22:27:28 +08:00
|
|
|
unsigned char *hpsizes, *lpsizes;
|
2018-03-07 09:37:12 +08:00
|
|
|
struct slice_mask *mask;
|
2018-03-07 09:37:09 +08:00
|
|
|
unsigned int psize = mmu_virtual_psize;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2018-03-07 09:37:09 +08:00
|
|
|
slice_dbg("slice_init_new_context_exec(mm=%p)\n", mm);
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2018-03-07 09:37:09 +08:00
|
|
|
/*
|
|
|
|
* In the case of exec, use the default limit. In the
|
|
|
|
* case of fork it is just inherited from the mm being
|
|
|
|
* duplicated.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_PPC64
|
2019-04-17 21:03:48 +08:00
|
|
|
mm_ctx_set_slb_addr_limit(&mm->context, DEFAULT_MAP_WINDOW_USER64);
|
2018-03-07 09:37:09 +08:00
|
|
|
#else
|
|
|
|
mm->context.slb_addr_limit = DEFAULT_MAP_WINDOW;
|
|
|
|
#endif
|
2019-04-17 21:03:48 +08:00
|
|
|
mm_ctx_set_user_psize(&mm->context, psize);
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2018-03-07 09:37:09 +08:00
|
|
|
/*
|
|
|
|
* Set all slice psizes to the default.
|
|
|
|
*/
|
2019-04-17 21:03:48 +08:00
|
|
|
lpsizes = mm_ctx_low_slices(&mm->context);
|
2018-03-07 09:37:09 +08:00
|
|
|
memset(lpsizes, (psize << 4) | psize, SLICE_NUM_LOW >> 1);
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2019-04-17 21:03:48 +08:00
|
|
|
hpsizes = mm_ctx_high_slices(&mm->context);
|
2018-03-07 09:37:09 +08:00
|
|
|
memset(hpsizes, (psize << 4) | psize, SLICE_NUM_HIGH >> 1);
|
2018-03-07 09:37:12 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Slice mask cache starts zeroed, fill the default size cache.
|
|
|
|
*/
|
2019-04-25 22:29:29 +08:00
|
|
|
mask = slice_mask_for_size(&mm->context, psize);
|
2018-03-07 09:37:12 +08:00
|
|
|
mask->low_slices = ~0UL;
|
|
|
|
if (SLICE_NUM_HIGH)
|
|
|
|
bitmap_fill(mask->high_slices, SLICE_NUM_HIGH);
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
|
2018-09-14 23:30:55 +08:00
|
|
|
#ifdef CONFIG_PPC_BOOK3S_64
|
|
|
|
void slice_setup_new_exec(void)
|
|
|
|
{
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
|
|
|
|
slice_dbg("slice_setup_new_exec(mm=%p)\n", mm);
|
|
|
|
|
|
|
|
if (!is_32bit_task())
|
|
|
|
return;
|
|
|
|
|
2019-04-17 21:03:48 +08:00
|
|
|
mm_ctx_set_slb_addr_limit(&mm->context, DEFAULT_MAP_WINDOW);
|
2018-09-14 23:30:55 +08:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2008-06-18 13:29:12 +08:00
|
|
|
void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
|
|
|
|
unsigned long len, unsigned int psize)
|
|
|
|
{
|
2017-03-22 11:36:48 +08:00
|
|
|
struct slice_mask mask;
|
2008-06-18 13:29:12 +08:00
|
|
|
|
2016-04-29 21:26:09 +08:00
|
|
|
VM_BUG_ON(radix_enabled());
|
2017-03-22 11:36:48 +08:00
|
|
|
|
|
|
|
slice_range_to_mask(start, len, &mask);
|
2018-03-07 09:37:11 +08:00
|
|
|
slice_convert(mm, &mask, psize);
|
2008-06-18 13:29:12 +08:00
|
|
|
}
|
|
|
|
|
2014-10-21 11:25:38 +08:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
2007-05-08 14:27:27 +08:00
|
|
|
/*
|
2012-09-20 09:48:00 +08:00
|
|
|
* is_hugepage_only_range() is used by generic code to verify whether
|
2007-05-08 14:27:27 +08:00
|
|
|
* a normal mmap mapping (non hugetlbfs) is valid on a given area.
|
|
|
|
*
|
|
|
|
* until the generic code provides a more generic hook and/or starts
|
|
|
|
* calling arch get_unmapped_area for MAP_FIXED (which our implementation
|
|
|
|
* here knows how to deal with), we hijack it to keep standard mappings
|
|
|
|
* away from us.
|
|
|
|
*
|
|
|
|
* because of that generic code limitation, MAP_FIXED mapping cannot
|
|
|
|
* "convert" back a slice with no VMAs to the standard page size, only
|
|
|
|
* get_unmapped_area() can. It would be possible to fix it here but I
|
|
|
|
* prefer working on fixing the generic code instead.
|
|
|
|
*
|
|
|
|
* WARNING: This will not work if hugetlbfs isn't enabled since the
|
|
|
|
* generic code will redefine that function as 0 in that. This is ok
|
|
|
|
* for now as we only use slices with hugetlbfs enabled. This should
|
|
|
|
* be fixed as the generic code gets fixed.
|
|
|
|
*/
|
2018-03-07 09:37:17 +08:00
|
|
|
int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,
|
2007-05-08 14:27:27 +08:00
|
|
|
unsigned long len)
|
|
|
|
{
|
2018-03-07 09:37:16 +08:00
|
|
|
const struct slice_mask *maskp;
|
2019-04-17 21:03:48 +08:00
|
|
|
unsigned int psize = mm_ctx_user_psize(&mm->context);
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2018-03-07 09:37:17 +08:00
|
|
|
VM_BUG_ON(radix_enabled());
|
2016-04-29 21:26:09 +08:00
|
|
|
|
2019-04-25 22:29:29 +08:00
|
|
|
maskp = slice_mask_for_size(&mm->context, psize);
|
2009-01-14 17:09:34 +08:00
|
|
|
#ifdef CONFIG_PPC_64K_PAGES
|
|
|
|
/* We need to account for 4k slices too */
|
|
|
|
if (psize == MMU_PAGE_64K) {
|
2018-03-07 09:37:16 +08:00
|
|
|
const struct slice_mask *compat_maskp;
|
|
|
|
struct slice_mask available;
|
|
|
|
|
2019-04-25 22:29:29 +08:00
|
|
|
compat_maskp = slice_mask_for_size(&mm->context, MMU_PAGE_4K);
|
2018-03-07 09:37:16 +08:00
|
|
|
slice_or_mask(&available, maskp, compat_maskp);
|
|
|
|
return !slice_check_range_fits(mm, &available, addr, len);
|
2009-01-14 17:09:34 +08:00
|
|
|
}
|
|
|
|
#endif
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2018-03-07 09:37:16 +08:00
|
|
|
return !slice_check_range_fits(mm, maskp, addr, len);
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
2014-10-21 11:25:38 +08:00
|
|
|
#endif
|