openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

percpu: relegate chunks unusable when failing small allocations 

In certain cases, requestors of percpu memory may want specific
alignments. However, it is possible to end up in situations where the
contig_hint matches, but the alignment does not. This causes excess
scanning of chunks that will fail. To prevent this, if a small
allocation fails (< 32B), the chunk is moved to the empty list. Once an
allocation is freed from that chunk, it is placed back into rotation.

Signed-off-by: Dennis Zhou <dennis@kernel.org>
Reviewed-by: Peng Fan <peng.fan@nxp.com>
This commit is contained in:
Dennis Zhou 2019-02-25 09:03:50 -08:00
parent 3e54097beb
commit 8744d85942
1 changed files with 26 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
mm/percpu.c
View File

@ -94,6 +94,8 @@
/* the slots are sorted by free bytes left, 1-31 bytes share the same slot */ /* the slots are sorted by free bytes left, 1-31 bytes share the same slot */
#define PCPU_SLOT_BASE_SHIFT 5 #define PCPU_SLOT_BASE_SHIFT 5
/* chunks in slots below this are subject to being sidelined on failed alloc */
#define PCPU_SLOT_FAIL_THRESHOLD 3
#define PCPU_EMPTY_POP_PAGES_LOW 2 #define PCPU_EMPTY_POP_PAGES_LOW 2
#define PCPU_EMPTY_POP_PAGES_HIGH 4 #define PCPU_EMPTY_POP_PAGES_HIGH 4
@ -488,6 +490,22 @@ static void pcpu_mem_free(void *ptr)
kvfree(ptr); kvfree(ptr);
} }
static void __pcpu_chunk_move(struct pcpu_chunk *chunk, int slot,
bool move_front)
{
if (chunk != pcpu_reserved_chunk) {
if (move_front)
list_move(&chunk->list, &pcpu_slot[slot]);
else
list_move_tail(&chunk->list, &pcpu_slot[slot]);
}
}
static void pcpu_chunk_move(struct pcpu_chunk *chunk, int slot)
{
__pcpu_chunk_move(chunk, slot, true);
}
/** /**
* pcpu_chunk_relocate - put chunk in the appropriate chunk slot * pcpu_chunk_relocate - put chunk in the appropriate chunk slot
* @chunk: chunk of interest * @chunk: chunk of interest
@ -505,12 +523,8 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
{ {
int nslot = pcpu_chunk_slot(chunk); int nslot = pcpu_chunk_slot(chunk);
if (chunk != pcpu_reserved_chunk && oslot != nslot) { if (oslot != nslot)
if (oslot < nslot) __pcpu_chunk_move(chunk, nslot, oslot < nslot);
list_move(&chunk->list, &pcpu_slot[nslot]);
else
list_move_tail(&chunk->list, &pcpu_slot[nslot]);
}
} }
/** /**
@ -1395,7 +1409,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved,
bool is_atomic = (gfp & GFP_KERNEL) != GFP_KERNEL; bool is_atomic = (gfp & GFP_KERNEL) != GFP_KERNEL;
bool do_warn = !(gfp & __GFP_NOWARN); bool do_warn = !(gfp & __GFP_NOWARN);
static int warn_limit = 10; static int warn_limit = 10;
struct pcpu_chunk *chunk; struct pcpu_chunk *chunk, *next;
const char *err; const char *err;
int slot, off, cpu, ret; int slot, off, cpu, ret;
unsigned long flags; unsigned long flags;
@ -1457,11 +1471,14 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved,
restart: restart:
/* search through normal chunks */ /* search through normal chunks */
for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) { for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) {
list_for_each_entry(chunk, &pcpu_slot[slot], list) { list_for_each_entry_safe(chunk, next, &pcpu_slot[slot], list) {
off = pcpu_find_block_fit(chunk, bits, bit_align, off = pcpu_find_block_fit(chunk, bits, bit_align,
is_atomic); is_atomic);
if (off < 0) if (off < 0) {
if (slot < PCPU_SLOT_FAIL_THRESHOLD)
pcpu_chunk_move(chunk, 0);
continue; continue;
}
off = pcpu_alloc_area(chunk, bits, bit_align, off); off = pcpu_alloc_area(chunk, bits, bit_align, off);
if (off >= 0) if (off >= 0)