2005-04-17 06:20:36 +08:00
|
|
|
#ifndef __LINUX_CPUMASK_H
|
|
|
|
#define __LINUX_CPUMASK_H
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Cpumasks provide a bitmap suitable for representing the
|
2009-09-24 23:34:53 +08:00
|
|
|
* set of CPU's in a system, one bit position per CPU number. In general,
|
|
|
|
* only nr_cpu_ids (<= NR_CPUS) bits are valid.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/threads.h>
|
|
|
|
#include <linux/bitmap.h>
|
2011-11-24 09:12:59 +08:00
|
|
|
#include <linux/bug.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2015-03-05 08:19:19 +08:00
|
|
|
/* Don't assign or return these: may not be this big! */
|
2008-11-05 10:39:10 +08:00
|
|
|
typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-12-30 06:35:15 +08:00
|
|
|
/**
|
2009-09-24 23:34:53 +08:00
|
|
|
* cpumask_bits - get the bits in a cpumask
|
|
|
|
* @maskp: the struct cpumask *
|
2008-12-30 06:35:15 +08:00
|
|
|
*
|
2009-09-24 23:34:53 +08:00
|
|
|
* You should only assume nr_cpu_ids bits of this mask are valid. This is
|
|
|
|
* a macro so it's const-correct.
|
2008-12-30 06:35:15 +08:00
|
|
|
*/
|
2009-09-24 23:34:53 +08:00
|
|
|
#define cpumask_bits(maskp) ((maskp)->bits)
|
mempolicy: add bitmap_onto() and bitmap_fold() operations
The following adds two more bitmap operators, bitmap_onto() and bitmap_fold(),
with the usual cpumask and nodemask wrappers.
The bitmap_onto() operator computes one bitmap relative to another. If the
n-th bit in the origin mask is set, then the m-th bit of the destination mask
will be set, where m is the position of the n-th set bit in the relative mask.
The bitmap_fold() operator folds a bitmap into a second that has bit m set iff
the input bitmap has some bit n set, where m == n mod sz, for the specified sz
value.
There are two substantive changes between this patch and its
predecessor bitmap_relative:
1) Renamed bitmap_relative() to be bitmap_onto().
2) Added bitmap_fold().
The essential motivation for bitmap_onto() is to provide a mechanism for
converting a cpuset-relative CPU or Node mask to an absolute mask. Cpuset
relative masks are written as if the current task were in a cpuset whose CPUs
or Nodes were just the consecutive ones numbered 0..N-1, for some N. The
bitmap_onto() operator is provided in anticipation of adding support for the
first such cpuset relative mask, by the mbind() and set_mempolicy() system
calls, using a planned flag of MPOL_F_RELATIVE_NODES. These bitmap operators
(and their nodemask wrappers, in particular) will be used in code that
converts the user specified cpuset relative memory policy to a specific system
node numbered policy, given the current mems_allowed of the tasks cpuset.
Such cpuset relative mempolicies will address two deficiencies
of the existing interface between cpusets and mempolicies:
1) A task cannot at present reliably establish a cpuset
relative mempolicy because there is an essential race
condition, in that the tasks cpuset may be changed in
between the time the task can query its cpuset placement,
and the time the task can issue the applicable mbind or
set_memplicy system call.
2) A task cannot at present establish what cpuset relative
mempolicy it would like to have, if it is in a smaller
cpuset than it might have mempolicy preferences for,
because the existing interface only allows specifying
mempolicies for nodes currently allowed by the cpuset.
Cpuset relative mempolicies are useful for tasks that don't distinguish
particularly between one CPU or Node and another, but only between how many of
each are allowed, and the proper placement of threads and memory pages on the
various CPUs and Nodes available.
The motivation for the added bitmap_fold() can be seen in the following
example.
Let's say an application has specified some mempolicies that presume 16 memory
nodes, including say a mempolicy that specified MPOL_F_RELATIVE_NODES (cpuset
relative) nodes 12-15. Then lets say that application is crammed into a
cpuset that only has 8 memory nodes, 0-7. If one just uses bitmap_onto(),
this mempolicy, mapped to that cpuset, would ignore the requested relative
nodes above 7, leaving it empty of nodes. That's not good; better to fold the
higher nodes down, so that some nodes are included in the resulting mapped
mempolicy. In this case, the mempolicy nodes 12-15 are taken modulo 8 (the
weight of the mems_allowed of the confining cpuset), resulting in a mempolicy
specifying nodes 4-7.
Signed-off-by: Paul Jackson <pj@sgi.com>
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: Andi Kleen <ak@suse.de>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: <kosaki.motohiro@jp.fujitsu.com>
Cc: <ray-lk@madrabbit.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 17:12:29 +08:00
|
|
|
|
2015-02-14 06:36:57 +08:00
|
|
|
/**
|
|
|
|
* cpumask_pr_args - printf args to output a cpumask
|
|
|
|
* @maskp: cpumask to be printed
|
|
|
|
*
|
|
|
|
* Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
|
|
|
|
*/
|
|
|
|
#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
|
|
|
|
|
2008-05-13 03:21:13 +08:00
|
|
|
#if NR_CPUS == 1
|
|
|
|
#define nr_cpu_ids 1
|
2009-09-24 23:34:53 +08:00
|
|
|
#else
|
2008-05-13 03:21:13 +08:00
|
|
|
extern int nr_cpu_ids;
|
|
|
|
#endif
|
|
|
|
|
2009-09-24 23:34:53 +08:00
|
|
|
#ifdef CONFIG_CPUMASK_OFFSTACK
|
|
|
|
/* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also,
|
|
|
|
* not all bits may be allocated. */
|
2017-05-09 06:56:15 +08:00
|
|
|
#define nr_cpumask_bits ((unsigned int)nr_cpu_ids)
|
2009-09-24 23:34:53 +08:00
|
|
|
#else
|
2017-05-09 06:56:15 +08:00
|
|
|
#define nr_cpumask_bits ((unsigned int)NR_CPUS)
|
2009-09-24 23:34:53 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The following particular system cpumasks and operations manage
|
2008-12-30 06:35:14 +08:00
|
|
|
* possible, present, active and online cpus.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2008-12-30 06:35:14 +08:00
|
|
|
* cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
|
|
|
|
* cpu_present_mask - has bit 'cpu' set iff cpu is populated
|
|
|
|
* cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
|
|
|
|
* cpu_active_mask - has bit 'cpu' set iff cpu available to migration
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2008-12-30 06:35:14 +08:00
|
|
|
* If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2008-12-30 06:35:14 +08:00
|
|
|
* The cpu_possible_mask is fixed at boot time, as the set of CPU id's
|
|
|
|
* that it is possible might ever be plugged in at anytime during the
|
|
|
|
* life of that system boot. The cpu_present_mask is dynamic(*),
|
|
|
|
* representing which CPUs are currently plugged in. And
|
|
|
|
* cpu_online_mask is the dynamic subset of cpu_present_mask,
|
|
|
|
* indicating those CPUs available for scheduling.
|
|
|
|
*
|
|
|
|
* If HOTPLUG is enabled, then cpu_possible_mask is forced to have
|
2005-04-17 06:20:36 +08:00
|
|
|
* all NR_CPUS bits set, otherwise it is just the set of CPUs that
|
|
|
|
* ACPI reports present at boot.
|
|
|
|
*
|
2008-12-30 06:35:14 +08:00
|
|
|
* If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
|
2005-04-17 06:20:36 +08:00
|
|
|
* depending on what ACPI reports as currently plugged in, otherwise
|
2008-12-30 06:35:14 +08:00
|
|
|
* cpu_present_mask is just a copy of cpu_possible_mask.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2008-12-30 06:35:14 +08:00
|
|
|
* (*) Well, cpu_present_mask is dynamic in the hotplug case. If not
|
|
|
|
* hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* Subtleties:
|
|
|
|
* 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
|
|
|
|
* assumption that their single CPU is online. The UP
|
2008-12-30 06:35:14 +08:00
|
|
|
* cpu_{online,possible,present}_masks are placebos. Changing them
|
2005-04-17 06:20:36 +08:00
|
|
|
* will have no useful affect on the following num_*_cpus()
|
|
|
|
* and cpu_*() macros in the UP case. This ugliness is a UP
|
|
|
|
* optimization - don't waste any instructions or memory references
|
|
|
|
* asking if you're online or how many CPUs there are if there is
|
|
|
|
* only one CPU.
|
|
|
|
*/
|
|
|
|
|
2016-01-21 07:00:19 +08:00
|
|
|
extern struct cpumask __cpu_possible_mask;
|
|
|
|
extern struct cpumask __cpu_online_mask;
|
|
|
|
extern struct cpumask __cpu_present_mask;
|
|
|
|
extern struct cpumask __cpu_active_mask;
|
2016-01-21 07:00:25 +08:00
|
|
|
#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
|
|
|
|
#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
|
|
|
|
#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
|
|
|
|
#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
|
2008-12-30 06:35:14 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#if NR_CPUS > 1
|
2008-12-30 06:35:15 +08:00
|
|
|
#define num_online_cpus() cpumask_weight(cpu_online_mask)
|
|
|
|
#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
|
|
|
|
#define num_present_cpus() cpumask_weight(cpu_present_mask)
|
2009-11-25 20:31:39 +08:00
|
|
|
#define num_active_cpus() cpumask_weight(cpu_active_mask)
|
2008-12-30 06:35:15 +08:00
|
|
|
#define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
|
|
|
|
#define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
|
|
|
|
#define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
|
|
|
|
#define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
|
2005-04-17 06:20:36 +08:00
|
|
|
#else
|
2010-03-06 05:42:41 +08:00
|
|
|
#define num_online_cpus() 1U
|
|
|
|
#define num_possible_cpus() 1U
|
|
|
|
#define num_present_cpus() 1U
|
|
|
|
#define num_active_cpus() 1U
|
2005-04-17 06:20:36 +08:00
|
|
|
#define cpu_online(cpu) ((cpu) == 0)
|
|
|
|
#define cpu_possible(cpu) ((cpu) == 0)
|
|
|
|
#define cpu_present(cpu) ((cpu) == 0)
|
2008-07-15 19:43:49 +08:00
|
|
|
#define cpu_active(cpu) ((cpu) == 0)
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
|
|
|
|
2008-11-05 10:39:10 +08:00
|
|
|
/* verify cpu argument to cpumask_* operators */
|
|
|
|
static inline unsigned int cpumask_check(unsigned int cpu)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
|
|
|
WARN_ON_ONCE(cpu >= nr_cpumask_bits);
|
|
|
|
#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
|
|
|
|
return cpu;
|
|
|
|
}
|
|
|
|
|
|
|
|
#if NR_CPUS == 1
|
2008-11-08 17:24:19 +08:00
|
|
|
/* Uniprocessor. Assume all masks are "1". */
|
|
|
|
static inline unsigned int cpumask_first(const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Valid inputs for n are -1 and 0. */
|
|
|
|
static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
return n+1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
return n+1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned int cpumask_next_and(int n,
|
|
|
|
const struct cpumask *srcp,
|
|
|
|
const struct cpumask *andp)
|
|
|
|
{
|
|
|
|
return n+1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* cpu must be a valid cpu, ie 0, so there's no other choice. */
|
|
|
|
static inline unsigned int cpumask_any_but(const struct cpumask *mask,
|
|
|
|
unsigned int cpu)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
2008-11-05 10:39:10 +08:00
|
|
|
|
cpumask_set_cpu_local_first => cpumask_local_spread, lament
da91309e0a7e (cpumask: Utility function to set n'th cpu...) created a
genuinely weird function. I never saw it before, it went through DaveM.
(He only does this to make us other maintainers feel better about our own
mistakes.)
cpumask_set_cpu_local_first's purpose is say "I need to spread things
across N online cpus, choose the ones on this numa node first"; you call
it in a loop.
It can fail. One of the two callers ignores this, the other aborts and
fails the device open.
It can fail in two ways: allocating the off-stack cpumask, or through a
convoluted codepath which AFAICT can only occur if cpu_online_mask
changes. Which shouldn't happen, because if cpu_online_mask can change
while you call this, it could return a now-offline cpu anyway.
It contains a nonsensical test "!cpumask_of_node(numa_node)". This was
drawn to my attention by Geert, who said this causes a warning on Sparc.
It sets a single bit in a cpumask instead of returning a cpu number,
because that's what the callers want.
It could be made more efficient by passing the previous cpu rather than
an index, but that would be more invasive to the callers.
Fixes: da91309e0a7e8966d916a74cce42ed170fde06bf
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au> (then rebased)
Tested-by: Amir Vadai <amirv@mellanox.com>
Acked-by: Amir Vadai <amirv@mellanox.com>
Acked-by: David S. Miller <davem@davemloft.net>
2015-05-09 01:44:13 +08:00
|
|
|
static inline unsigned int cpumask_local_spread(unsigned int i, int node)
|
cpumask: Utility function to set n'th cpu - local cpu first
This function sets the n'th cpu - local cpu's first.
For example: in a 16 cores server with even cpu's local, will get the
following values:
cpumask_set_cpu_local_first(0, numa, cpumask) => cpu 0 is set
cpumask_set_cpu_local_first(1, numa, cpumask) => cpu 2 is set
...
cpumask_set_cpu_local_first(7, numa, cpumask) => cpu 14 is set
cpumask_set_cpu_local_first(8, numa, cpumask) => cpu 1 is set
cpumask_set_cpu_local_first(9, numa, cpumask) => cpu 3 is set
...
cpumask_set_cpu_local_first(15, numa, cpumask) => cpu 15 is set
Curently this function will be used by multi queue networking devices to
calculate the irq affinity mask, such that as many local cpu's as
possible will be utilized to handle the mq device irq's.
Signed-off-by: Amir Vadai <amirv@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-06-09 15:24:38 +08:00
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-11-05 10:39:10 +08:00
|
|
|
#define for_each_cpu(cpu, mask) \
|
|
|
|
for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
|
2010-02-23 09:04:59 +08:00
|
|
|
#define for_each_cpu_not(cpu, mask) \
|
|
|
|
for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
|
2008-11-05 10:39:10 +08:00
|
|
|
#define for_each_cpu_and(cpu, mask, and) \
|
|
|
|
for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and)
|
|
|
|
#else
|
|
|
|
/**
|
|
|
|
* cpumask_first - get the first cpu in a cpumask
|
|
|
|
* @srcp: the cpumask pointer
|
|
|
|
*
|
|
|
|
* Returns >= nr_cpu_ids if no cpus set.
|
|
|
|
*/
|
|
|
|
static inline unsigned int cpumask_first(const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_next - get the next cpu in a cpumask
|
|
|
|
* @n: the cpu prior to the place to search (ie. return will be > @n)
|
|
|
|
* @srcp: the cpumask pointer
|
|
|
|
*
|
|
|
|
* Returns >= nr_cpu_ids if no further cpus set.
|
|
|
|
*/
|
|
|
|
static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
/* -1 is a legal arg here. */
|
|
|
|
if (n != -1)
|
|
|
|
cpumask_check(n);
|
|
|
|
return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_next_zero - get the next unset cpu in a cpumask
|
|
|
|
* @n: the cpu prior to the place to search (ie. return will be > @n)
|
|
|
|
* @srcp: the cpumask pointer
|
|
|
|
*
|
|
|
|
* Returns >= nr_cpu_ids if no further cpus unset.
|
|
|
|
*/
|
|
|
|
static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
/* -1 is a legal arg here. */
|
|
|
|
if (n != -1)
|
|
|
|
cpumask_check(n);
|
|
|
|
return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
|
|
|
|
}
|
|
|
|
|
|
|
|
int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
|
|
|
|
int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
|
cpumask_set_cpu_local_first => cpumask_local_spread, lament
da91309e0a7e (cpumask: Utility function to set n'th cpu...) created a
genuinely weird function. I never saw it before, it went through DaveM.
(He only does this to make us other maintainers feel better about our own
mistakes.)
cpumask_set_cpu_local_first's purpose is say "I need to spread things
across N online cpus, choose the ones on this numa node first"; you call
it in a loop.
It can fail. One of the two callers ignores this, the other aborts and
fails the device open.
It can fail in two ways: allocating the off-stack cpumask, or through a
convoluted codepath which AFAICT can only occur if cpu_online_mask
changes. Which shouldn't happen, because if cpu_online_mask can change
while you call this, it could return a now-offline cpu anyway.
It contains a nonsensical test "!cpumask_of_node(numa_node)". This was
drawn to my attention by Geert, who said this causes a warning on Sparc.
It sets a single bit in a cpumask instead of returning a cpu number,
because that's what the callers want.
It could be made more efficient by passing the previous cpu rather than
an index, but that would be more invasive to the callers.
Fixes: da91309e0a7e8966d916a74cce42ed170fde06bf
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au> (then rebased)
Tested-by: Amir Vadai <amirv@mellanox.com>
Acked-by: Amir Vadai <amirv@mellanox.com>
Acked-by: David S. Miller <davem@davemloft.net>
2015-05-09 01:44:13 +08:00
|
|
|
unsigned int cpumask_local_spread(unsigned int i, int node);
|
2008-11-05 10:39:10 +08:00
|
|
|
|
2008-11-08 17:24:19 +08:00
|
|
|
/**
|
|
|
|
* for_each_cpu - iterate over every cpu in a mask
|
|
|
|
* @cpu: the (optionally unsigned) integer iterator
|
|
|
|
* @mask: the cpumask pointer
|
|
|
|
*
|
|
|
|
* After the loop, cpu is >= nr_cpu_ids.
|
|
|
|
*/
|
2008-11-05 10:39:10 +08:00
|
|
|
#define for_each_cpu(cpu, mask) \
|
|
|
|
for ((cpu) = -1; \
|
|
|
|
(cpu) = cpumask_next((cpu), (mask)), \
|
|
|
|
(cpu) < nr_cpu_ids;)
|
2008-11-08 17:24:19 +08:00
|
|
|
|
2010-02-23 09:04:59 +08:00
|
|
|
/**
|
|
|
|
* for_each_cpu_not - iterate over every cpu in a complemented mask
|
|
|
|
* @cpu: the (optionally unsigned) integer iterator
|
|
|
|
* @mask: the cpumask pointer
|
|
|
|
*
|
|
|
|
* After the loop, cpu is >= nr_cpu_ids.
|
|
|
|
*/
|
|
|
|
#define for_each_cpu_not(cpu, mask) \
|
|
|
|
for ((cpu) = -1; \
|
|
|
|
(cpu) = cpumask_next_zero((cpu), (mask)), \
|
|
|
|
(cpu) < nr_cpu_ids;)
|
|
|
|
|
2008-11-08 17:24:19 +08:00
|
|
|
/**
|
|
|
|
* for_each_cpu_and - iterate over every cpu in both masks
|
|
|
|
* @cpu: the (optionally unsigned) integer iterator
|
|
|
|
* @mask: the first cpumask pointer
|
|
|
|
* @and: the second cpumask pointer
|
|
|
|
*
|
|
|
|
* This saves a temporary CPU mask in many places. It is equivalent to:
|
|
|
|
* struct cpumask tmp;
|
|
|
|
* cpumask_and(&tmp, &mask, &and);
|
|
|
|
* for_each_cpu(cpu, &tmp)
|
|
|
|
* ...
|
|
|
|
*
|
|
|
|
* After the loop, cpu is >= nr_cpu_ids.
|
|
|
|
*/
|
2008-11-05 10:39:10 +08:00
|
|
|
#define for_each_cpu_and(cpu, mask, and) \
|
|
|
|
for ((cpu) = -1; \
|
|
|
|
(cpu) = cpumask_next_and((cpu), (mask), (and)), \
|
|
|
|
(cpu) < nr_cpu_ids;)
|
|
|
|
#endif /* SMP */
|
|
|
|
|
|
|
|
#define CPU_BITS_NONE \
|
|
|
|
{ \
|
|
|
|
[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
|
|
|
|
}
|
|
|
|
|
|
|
|
#define CPU_BITS_CPU0 \
|
|
|
|
{ \
|
|
|
|
[0] = 1UL \
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_set_cpu - set a cpu in a cpumask
|
|
|
|
* @cpu: cpu number (< nr_cpu_ids)
|
|
|
|
* @dstp: the cpumask pointer
|
|
|
|
*/
|
|
|
|
static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
|
|
|
|
{
|
|
|
|
set_bit(cpumask_check(cpu), cpumask_bits(dstp));
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_clear_cpu - clear a cpu in a cpumask
|
|
|
|
* @cpu: cpu number (< nr_cpu_ids)
|
|
|
|
* @dstp: the cpumask pointer
|
|
|
|
*/
|
|
|
|
static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
|
|
|
|
{
|
|
|
|
clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_test_cpu - test for a cpu in a cpumask
|
|
|
|
* @cpu: cpu number (< nr_cpu_ids)
|
|
|
|
* @cpumask: the cpumask pointer
|
|
|
|
*
|
2012-05-28 22:23:51 +08:00
|
|
|
* Returns 1 if @cpu is set in @cpumask, else returns 0
|
2008-11-05 10:39:10 +08:00
|
|
|
*/
|
2015-03-31 10:55:05 +08:00
|
|
|
static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
|
|
|
|
{
|
|
|
|
return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
|
|
|
|
}
|
2008-11-05 10:39:10 +08:00
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
|
|
|
|
* @cpu: cpu number (< nr_cpu_ids)
|
|
|
|
* @cpumask: the cpumask pointer
|
|
|
|
*
|
2012-05-28 22:23:51 +08:00
|
|
|
* Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
|
|
|
|
*
|
2008-11-05 10:39:10 +08:00
|
|
|
* test_and_set_bit wrapper for cpumasks.
|
|
|
|
*/
|
|
|
|
static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
|
|
|
|
{
|
|
|
|
return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
|
|
|
|
}
|
|
|
|
|
generic-ipi: make struct call_function_data lockless
This patch can remove spinlock from struct call_function_data, the
reasons are below:
1: add a new interface for cpumask named cpumask_test_and_clear_cpu(),
it can atomically test and clear specific cpu, we can use it instead
of cpumask_test_cpu() and cpumask_clear_cpu() and no need data->lock
to protect those in generic_smp_call_function_interrupt().
2: in smp_call_function_many(), after csd_lock() return, the current's
cfd_data is deleted from call_function list, so it not have race
between other cpus, then cfs_data is only used in
smp_call_function_many() that must disable preemption and not from
a hardware interrupthandler or from a bottom half handler to call,
only the correspond cpu can use it, so it not have race in current
cpu, no need cfs_data->lock to protect it.
3: after 1 and 2, cfs_data->lock is only use to protect cfs_data->refs in
generic_smp_call_function_interrupt(), so we can define cfs_data->refs
to atomic_t, and no need cfs_data->lock any more.
Signed-off-by: Xiao Guangrong <xiaoguangrong@cn.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Jens Axboe <jens.axboe@oracle.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Peter Zijlstra <peterz@infradead.org>
Acked-by: Rusty Russell <rusty@rustcorp.com.au>
[akpm@linux-foundation.org: use atomic_dec_return()]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-23 07:43:39 +08:00
|
|
|
/**
|
|
|
|
* cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
|
|
|
|
* @cpu: cpu number (< nr_cpu_ids)
|
|
|
|
* @cpumask: the cpumask pointer
|
|
|
|
*
|
2012-05-28 22:23:51 +08:00
|
|
|
* Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
|
|
|
|
*
|
generic-ipi: make struct call_function_data lockless
This patch can remove spinlock from struct call_function_data, the
reasons are below:
1: add a new interface for cpumask named cpumask_test_and_clear_cpu(),
it can atomically test and clear specific cpu, we can use it instead
of cpumask_test_cpu() and cpumask_clear_cpu() and no need data->lock
to protect those in generic_smp_call_function_interrupt().
2: in smp_call_function_many(), after csd_lock() return, the current's
cfd_data is deleted from call_function list, so it not have race
between other cpus, then cfs_data is only used in
smp_call_function_many() that must disable preemption and not from
a hardware interrupthandler or from a bottom half handler to call,
only the correspond cpu can use it, so it not have race in current
cpu, no need cfs_data->lock to protect it.
3: after 1 and 2, cfs_data->lock is only use to protect cfs_data->refs in
generic_smp_call_function_interrupt(), so we can define cfs_data->refs
to atomic_t, and no need cfs_data->lock any more.
Signed-off-by: Xiao Guangrong <xiaoguangrong@cn.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Jens Axboe <jens.axboe@oracle.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Peter Zijlstra <peterz@infradead.org>
Acked-by: Rusty Russell <rusty@rustcorp.com.au>
[akpm@linux-foundation.org: use atomic_dec_return()]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-23 07:43:39 +08:00
|
|
|
* test_and_clear_bit wrapper for cpumasks.
|
|
|
|
*/
|
|
|
|
static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
|
|
|
|
{
|
|
|
|
return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
|
|
|
|
}
|
|
|
|
|
2008-11-05 10:39:10 +08:00
|
|
|
/**
|
|
|
|
* cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
|
|
|
|
* @dstp: the cpumask pointer
|
|
|
|
*/
|
|
|
|
static inline void cpumask_setall(struct cpumask *dstp)
|
|
|
|
{
|
|
|
|
bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
|
|
|
|
* @dstp: the cpumask pointer
|
|
|
|
*/
|
|
|
|
static inline void cpumask_clear(struct cpumask *dstp)
|
|
|
|
{
|
|
|
|
bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_and - *dstp = *src1p & *src2p
|
|
|
|
* @dstp: the cpumask result
|
|
|
|
* @src1p: the first input
|
|
|
|
* @src2p: the second input
|
2012-05-28 22:23:51 +08:00
|
|
|
*
|
|
|
|
* If *@dstp is empty, returns 0, else returns 1
|
2008-11-05 10:39:10 +08:00
|
|
|
*/
|
2009-08-22 00:26:15 +08:00
|
|
|
static inline int cpumask_and(struct cpumask *dstp,
|
2008-11-05 10:39:10 +08:00
|
|
|
const struct cpumask *src1p,
|
|
|
|
const struct cpumask *src2p)
|
|
|
|
{
|
2009-08-22 00:26:15 +08:00
|
|
|
return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
|
2008-11-05 10:39:10 +08:00
|
|
|
cpumask_bits(src2p), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_or - *dstp = *src1p | *src2p
|
|
|
|
* @dstp: the cpumask result
|
|
|
|
* @src1p: the first input
|
|
|
|
* @src2p: the second input
|
|
|
|
*/
|
|
|
|
static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
|
|
|
|
const struct cpumask *src2p)
|
|
|
|
{
|
|
|
|
bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
|
|
|
|
cpumask_bits(src2p), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_xor - *dstp = *src1p ^ *src2p
|
|
|
|
* @dstp: the cpumask result
|
|
|
|
* @src1p: the first input
|
|
|
|
* @src2p: the second input
|
|
|
|
*/
|
|
|
|
static inline void cpumask_xor(struct cpumask *dstp,
|
|
|
|
const struct cpumask *src1p,
|
|
|
|
const struct cpumask *src2p)
|
|
|
|
{
|
|
|
|
bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
|
|
|
|
cpumask_bits(src2p), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_andnot - *dstp = *src1p & ~*src2p
|
|
|
|
* @dstp: the cpumask result
|
|
|
|
* @src1p: the first input
|
|
|
|
* @src2p: the second input
|
2012-05-28 22:23:51 +08:00
|
|
|
*
|
|
|
|
* If *@dstp is empty, returns 0, else returns 1
|
2008-11-05 10:39:10 +08:00
|
|
|
*/
|
2009-08-22 00:26:15 +08:00
|
|
|
static inline int cpumask_andnot(struct cpumask *dstp,
|
2008-11-05 10:39:10 +08:00
|
|
|
const struct cpumask *src1p,
|
|
|
|
const struct cpumask *src2p)
|
|
|
|
{
|
2009-08-22 00:26:15 +08:00
|
|
|
return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
|
2008-11-05 10:39:10 +08:00
|
|
|
cpumask_bits(src2p), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_complement - *dstp = ~*srcp
|
|
|
|
* @dstp: the cpumask result
|
|
|
|
* @srcp: the input to invert
|
|
|
|
*/
|
|
|
|
static inline void cpumask_complement(struct cpumask *dstp,
|
|
|
|
const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
|
|
|
|
nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_equal - *src1p == *src2p
|
|
|
|
* @src1p: the first input
|
|
|
|
* @src2p: the second input
|
|
|
|
*/
|
|
|
|
static inline bool cpumask_equal(const struct cpumask *src1p,
|
|
|
|
const struct cpumask *src2p)
|
|
|
|
{
|
|
|
|
return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
|
|
|
|
nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_intersects - (*src1p & *src2p) != 0
|
|
|
|
* @src1p: the first input
|
|
|
|
* @src2p: the second input
|
|
|
|
*/
|
|
|
|
static inline bool cpumask_intersects(const struct cpumask *src1p,
|
|
|
|
const struct cpumask *src2p)
|
|
|
|
{
|
|
|
|
return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
|
|
|
|
nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_subset - (*src1p & ~*src2p) == 0
|
|
|
|
* @src1p: the first input
|
|
|
|
* @src2p: the second input
|
2012-05-28 22:23:51 +08:00
|
|
|
*
|
|
|
|
* Returns 1 if *@src1p is a subset of *@src2p, else returns 0
|
2008-11-05 10:39:10 +08:00
|
|
|
*/
|
|
|
|
static inline int cpumask_subset(const struct cpumask *src1p,
|
|
|
|
const struct cpumask *src2p)
|
|
|
|
{
|
|
|
|
return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
|
|
|
|
nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_empty - *srcp == 0
|
|
|
|
* @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
|
|
|
|
*/
|
|
|
|
static inline bool cpumask_empty(const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_full - *srcp == 0xFFFFFFFF...
|
|
|
|
* @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
|
|
|
|
*/
|
|
|
|
static inline bool cpumask_full(const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_weight - Count of bits in *srcp
|
|
|
|
* @srcp: the cpumask to count bits (< nr_cpu_ids) in.
|
|
|
|
*/
|
|
|
|
static inline unsigned int cpumask_weight(const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_shift_right - *dstp = *srcp >> n
|
|
|
|
* @dstp: the cpumask result
|
|
|
|
* @srcp: the input to shift
|
|
|
|
* @n: the number of bits to shift by
|
|
|
|
*/
|
|
|
|
static inline void cpumask_shift_right(struct cpumask *dstp,
|
|
|
|
const struct cpumask *srcp, int n)
|
|
|
|
{
|
|
|
|
bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
|
|
|
|
nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_shift_left - *dstp = *srcp << n
|
|
|
|
* @dstp: the cpumask result
|
|
|
|
* @srcp: the input to shift
|
|
|
|
* @n: the number of bits to shift by
|
|
|
|
*/
|
|
|
|
static inline void cpumask_shift_left(struct cpumask *dstp,
|
|
|
|
const struct cpumask *srcp, int n)
|
|
|
|
{
|
|
|
|
bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
|
|
|
|
nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_copy - *dstp = *srcp
|
|
|
|
* @dstp: the result
|
|
|
|
* @srcp: the input cpumask
|
|
|
|
*/
|
|
|
|
static inline void cpumask_copy(struct cpumask *dstp,
|
|
|
|
const struct cpumask *srcp)
|
|
|
|
{
|
|
|
|
bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_any - pick a "random" cpu from *srcp
|
|
|
|
* @srcp: the input cpumask
|
|
|
|
*
|
|
|
|
* Returns >= nr_cpu_ids if no cpus set.
|
|
|
|
*/
|
|
|
|
#define cpumask_any(srcp) cpumask_first(srcp)
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_first_and - return the first cpu from *srcp1 & *srcp2
|
|
|
|
* @src1p: the first input
|
|
|
|
* @src2p: the second input
|
|
|
|
*
|
|
|
|
* Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
|
|
|
|
*/
|
|
|
|
#define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
|
|
|
|
* @mask1: the first input cpumask
|
|
|
|
* @mask2: the second input cpumask
|
|
|
|
*
|
|
|
|
* Returns >= nr_cpu_ids if no cpus set.
|
|
|
|
*/
|
|
|
|
#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
|
|
|
|
|
2008-11-07 08:12:29 +08:00
|
|
|
/**
|
|
|
|
* cpumask_of - the cpumask containing just a given cpu
|
|
|
|
* @cpu: the cpu (<= nr_cpu_ids)
|
|
|
|
*/
|
|
|
|
#define cpumask_of(cpu) (get_cpu_mask(cpu))
|
|
|
|
|
2008-12-13 18:50:25 +08:00
|
|
|
/**
|
|
|
|
* cpumask_parse_user - extract a cpumask from a user string
|
|
|
|
* @buf: the buffer to extract from
|
|
|
|
* @len: the length of the buffer
|
|
|
|
* @dstp: the cpumask to set.
|
|
|
|
*
|
|
|
|
* Returns -errno, or 0 for success.
|
|
|
|
*/
|
|
|
|
static inline int cpumask_parse_user(const char __user *buf, int len,
|
|
|
|
struct cpumask *dstp)
|
|
|
|
{
|
2017-02-09 06:30:56 +08:00
|
|
|
return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
|
2008-12-13 18:50:25 +08:00
|
|
|
}
|
|
|
|
|
bitmap, irq: add smp_affinity_list interface to /proc/irq
Manually adjusting the smp_affinity for IRQ's becomes unwieldy when the
cpu count is large.
Setting smp affinity to cpus 256 to 263 would be:
echo 000000ff,00000000,00000000,00000000,00000000,00000000,00000000,00000000 > smp_affinity
instead of:
echo 256-263 > smp_affinity_list
Think about what it looks like for cpus around say, 4088 to 4095.
We already have many alternate "list" interfaces:
/sys/devices/system/cpu/cpuX/indexY/shared_cpu_list
/sys/devices/system/cpu/cpuX/topology/thread_siblings_list
/sys/devices/system/cpu/cpuX/topology/core_siblings_list
/sys/devices/system/node/nodeX/cpulist
/sys/devices/pci***/***/local_cpulist
Add a companion interface, smp_affinity_list to use cpu lists instead of
cpu maps. This conforms to other companion interfaces where both a map
and a list interface exists.
This required adding a bitmap_parselist_user() function in a manner
similar to the bitmap_parse_user() function.
[akpm@linux-foundation.org: make __bitmap_parselist() static]
Signed-off-by: Mike Travis <travis@sgi.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Jack Steiner <steiner@sgi.com>
Cc: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Andy Shevchenko <andy.shevchenko@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-05-25 08:13:12 +08:00
|
|
|
/**
|
|
|
|
* cpumask_parselist_user - extract a cpumask from a user string
|
|
|
|
* @buf: the buffer to extract from
|
|
|
|
* @len: the length of the buffer
|
|
|
|
* @dstp: the cpumask to set.
|
|
|
|
*
|
|
|
|
* Returns -errno, or 0 for success.
|
|
|
|
*/
|
|
|
|
static inline int cpumask_parselist_user(const char __user *buf, int len,
|
|
|
|
struct cpumask *dstp)
|
|
|
|
{
|
|
|
|
return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
|
2017-02-09 06:30:56 +08:00
|
|
|
nr_cpumask_bits);
|
bitmap, irq: add smp_affinity_list interface to /proc/irq
Manually adjusting the smp_affinity for IRQ's becomes unwieldy when the
cpu count is large.
Setting smp affinity to cpus 256 to 263 would be:
echo 000000ff,00000000,00000000,00000000,00000000,00000000,00000000,00000000 > smp_affinity
instead of:
echo 256-263 > smp_affinity_list
Think about what it looks like for cpus around say, 4088 to 4095.
We already have many alternate "list" interfaces:
/sys/devices/system/cpu/cpuX/indexY/shared_cpu_list
/sys/devices/system/cpu/cpuX/topology/thread_siblings_list
/sys/devices/system/cpu/cpuX/topology/core_siblings_list
/sys/devices/system/node/nodeX/cpulist
/sys/devices/pci***/***/local_cpulist
Add a companion interface, smp_affinity_list to use cpu lists instead of
cpu maps. This conforms to other companion interfaces where both a map
and a list interface exists.
This required adding a bitmap_parselist_user() function in a manner
similar to the bitmap_parse_user() function.
[akpm@linux-foundation.org: make __bitmap_parselist() static]
Signed-off-by: Mike Travis <travis@sgi.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Jack Steiner <steiner@sgi.com>
Cc: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Andy Shevchenko <andy.shevchenko@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-05-25 08:13:12 +08:00
|
|
|
}
|
|
|
|
|
2013-03-13 02:30:04 +08:00
|
|
|
/**
|
2016-08-03 05:05:42 +08:00
|
|
|
* cpumask_parse - extract a cpumask from a string
|
2013-03-13 02:30:04 +08:00
|
|
|
* @buf: the buffer to extract from
|
|
|
|
* @dstp: the cpumask to set.
|
|
|
|
*
|
|
|
|
* Returns -errno, or 0 for success.
|
|
|
|
*/
|
|
|
|
static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
|
|
|
|
{
|
|
|
|
char *nl = strchr(buf, '\n');
|
2014-05-14 09:03:45 +08:00
|
|
|
unsigned int len = nl ? (unsigned int)(nl - buf) : strlen(buf);
|
2013-03-13 02:30:04 +08:00
|
|
|
|
2017-02-09 06:30:56 +08:00
|
|
|
return bitmap_parse(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
|
2013-03-13 02:30:04 +08:00
|
|
|
}
|
|
|
|
|
2008-12-13 18:50:25 +08:00
|
|
|
/**
|
2012-07-27 07:59:42 +08:00
|
|
|
* cpulist_parse - extract a cpumask from a user string of ranges
|
2008-12-13 18:50:25 +08:00
|
|
|
* @buf: the buffer to extract from
|
|
|
|
* @dstp: the cpumask to set.
|
|
|
|
*
|
|
|
|
* Returns -errno, or 0 for success.
|
|
|
|
*/
|
|
|
|
static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
|
|
|
|
{
|
2017-02-09 06:30:56 +08:00
|
|
|
return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
|
2008-11-05 10:39:10 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cpumask_size - size to allocate for a 'struct cpumask' in bytes
|
|
|
|
*/
|
|
|
|
static inline size_t cpumask_size(void)
|
|
|
|
{
|
2015-03-05 08:19:19 +08:00
|
|
|
return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
|
2008-11-05 10:39:10 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* cpumask_var_t: struct cpumask for stack usage.
|
|
|
|
*
|
|
|
|
* Oh, the wicked games we play! In order to make kernel coding a
|
|
|
|
* little more difficult, we typedef cpumask_var_t to an array or a
|
|
|
|
* pointer: doing &mask on an array is a noop, so it still works.
|
|
|
|
*
|
|
|
|
* ie.
|
|
|
|
* cpumask_var_t tmpmask;
|
|
|
|
* if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
|
|
|
|
* return -ENOMEM;
|
|
|
|
*
|
|
|
|
* ... use 'tmpmask' like a normal struct cpumask * ...
|
|
|
|
*
|
|
|
|
* free_cpumask_var(tmpmask);
|
2011-07-27 07:08:45 +08:00
|
|
|
*
|
|
|
|
*
|
|
|
|
* However, one notable exception is there. alloc_cpumask_var() allocates
|
|
|
|
* only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
|
|
|
|
* NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
|
|
|
|
*
|
|
|
|
* cpumask_var_t tmpmask;
|
|
|
|
* if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
|
|
|
|
* return -ENOMEM;
|
|
|
|
*
|
|
|
|
* var = *tmpmask;
|
|
|
|
*
|
|
|
|
* This code makes NR_CPUS length memcopy and brings to a memory corruption.
|
|
|
|
* cpumask_copy() provide safe copy functionality.
|
2014-08-27 08:12:21 +08:00
|
|
|
*
|
|
|
|
* Note that there is another evil here: If you define a cpumask_var_t
|
|
|
|
* as a percpu variable then the way to obtain the address of the cpumask
|
|
|
|
* structure differently influences what this_cpu_* operation needs to be
|
|
|
|
* used. Please use this_cpu_cpumask_var_t in those cases. The direct use
|
|
|
|
* of this_cpu_ptr() or this_cpu_read() will lead to failures when the
|
|
|
|
* other type of cpumask_var_t implementation is configured.
|
2017-01-31 01:57:43 +08:00
|
|
|
*
|
|
|
|
* Please also note that __cpumask_var_read_mostly can be used to declare
|
|
|
|
* a cpumask_var_t variable itself (not its content) as read mostly.
|
2008-11-05 10:39:10 +08:00
|
|
|
*/
|
|
|
|
#ifdef CONFIG_CPUMASK_OFFSTACK
|
|
|
|
typedef struct cpumask *cpumask_var_t;
|
|
|
|
|
2017-01-31 01:57:43 +08:00
|
|
|
#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
|
|
|
|
#define __cpumask_var_read_mostly __read_mostly
|
2014-08-27 08:12:21 +08:00
|
|
|
|
2008-12-19 14:26:37 +08:00
|
|
|
bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
|
2008-11-05 10:39:10 +08:00
|
|
|
bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
|
2009-06-07 05:50:36 +08:00
|
|
|
bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
|
|
|
|
bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
|
2008-11-05 10:39:10 +08:00
|
|
|
void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
|
|
|
|
void free_cpumask_var(cpumask_var_t mask);
|
2008-11-07 08:12:29 +08:00
|
|
|
void free_bootmem_cpumask_var(cpumask_var_t mask);
|
2008-11-05 10:39:10 +08:00
|
|
|
|
2017-04-13 02:20:29 +08:00
|
|
|
static inline bool cpumask_available(cpumask_var_t mask)
|
|
|
|
{
|
|
|
|
return mask != NULL;
|
|
|
|
}
|
|
|
|
|
2008-11-05 10:39:10 +08:00
|
|
|
#else
|
|
|
|
typedef struct cpumask cpumask_var_t[1];
|
|
|
|
|
2014-08-27 08:12:21 +08:00
|
|
|
#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
|
2017-01-31 01:57:43 +08:00
|
|
|
#define __cpumask_var_read_mostly
|
2014-08-27 08:12:21 +08:00
|
|
|
|
2008-11-05 10:39:10 +08:00
|
|
|
static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2008-12-19 14:26:37 +08:00
|
|
|
static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
|
|
|
|
int node)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2009-06-07 05:50:36 +08:00
|
|
|
static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
|
|
|
{
|
|
|
|
cpumask_clear(*mask);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
|
|
|
|
int node)
|
|
|
|
{
|
|
|
|
cpumask_clear(*mask);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2008-11-05 10:39:10 +08:00
|
|
|
static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void free_cpumask_var(cpumask_var_t mask)
|
|
|
|
{
|
|
|
|
}
|
2008-11-07 08:12:29 +08:00
|
|
|
|
|
|
|
static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
|
|
|
|
{
|
|
|
|
}
|
2017-04-13 02:20:29 +08:00
|
|
|
|
|
|
|
static inline bool cpumask_available(cpumask_var_t mask)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
2008-11-05 10:39:10 +08:00
|
|
|
#endif /* CONFIG_CPUMASK_OFFSTACK */
|
|
|
|
|
|
|
|
/* It's common to want to use cpu_all_mask in struct member initializers,
|
|
|
|
* so it has to refer to an address rather than a pointer. */
|
|
|
|
extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
|
|
|
|
#define cpu_all_mask to_cpumask(cpu_all_bits)
|
|
|
|
|
|
|
|
/* First bits of cpu_bit_bitmap are in fact unset. */
|
|
|
|
#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
|
|
|
|
|
2008-12-30 06:35:15 +08:00
|
|
|
#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
|
|
|
|
#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
|
|
|
|
#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
|
|
|
|
|
2008-11-05 10:39:10 +08:00
|
|
|
/* Wrappers for arch boot code to manipulate normally-constant masks */
|
2008-12-30 06:35:16 +08:00
|
|
|
void init_cpu_present(const struct cpumask *src);
|
|
|
|
void init_cpu_possible(const struct cpumask *src);
|
|
|
|
void init_cpu_online(const struct cpumask *src);
|
2009-09-24 23:34:53 +08:00
|
|
|
|
2016-12-14 02:32:28 +08:00
|
|
|
static inline void reset_cpu_possible_mask(void)
|
|
|
|
{
|
|
|
|
bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
|
|
|
|
}
|
|
|
|
|
2016-01-21 07:00:28 +08:00
|
|
|
static inline void
|
|
|
|
set_cpu_possible(unsigned int cpu, bool possible)
|
|
|
|
{
|
|
|
|
if (possible)
|
|
|
|
cpumask_set_cpu(cpu, &__cpu_possible_mask);
|
|
|
|
else
|
|
|
|
cpumask_clear_cpu(cpu, &__cpu_possible_mask);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
set_cpu_present(unsigned int cpu, bool present)
|
|
|
|
{
|
|
|
|
if (present)
|
|
|
|
cpumask_set_cpu(cpu, &__cpu_present_mask);
|
|
|
|
else
|
|
|
|
cpumask_clear_cpu(cpu, &__cpu_present_mask);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
set_cpu_online(unsigned int cpu, bool online)
|
|
|
|
{
|
2016-03-10 19:54:08 +08:00
|
|
|
if (online)
|
2016-01-21 07:00:28 +08:00
|
|
|
cpumask_set_cpu(cpu, &__cpu_online_mask);
|
2016-03-10 19:54:08 +08:00
|
|
|
else
|
2016-01-21 07:00:28 +08:00
|
|
|
cpumask_clear_cpu(cpu, &__cpu_online_mask);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
set_cpu_active(unsigned int cpu, bool active)
|
|
|
|
{
|
|
|
|
if (active)
|
|
|
|
cpumask_set_cpu(cpu, &__cpu_active_mask);
|
|
|
|
else
|
|
|
|
cpumask_clear_cpu(cpu, &__cpu_active_mask);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2009-09-24 23:34:53 +08:00
|
|
|
/**
|
|
|
|
* to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
|
|
|
|
* @bitmap: the bitmap
|
|
|
|
*
|
|
|
|
* There are a few places where cpumask_var_t isn't appropriate and
|
|
|
|
* static cpumasks must be used (eg. very early boot), yet we don't
|
|
|
|
* expose the definition of 'struct cpumask'.
|
|
|
|
*
|
|
|
|
* This does the conversion, and can be used as a constant initializer.
|
|
|
|
*/
|
|
|
|
#define to_cpumask(bitmap) \
|
|
|
|
((struct cpumask *)(1 ? (bitmap) \
|
|
|
|
: (void *)sizeof(__check_is_bitmap(bitmap))))
|
|
|
|
|
|
|
|
static inline int __check_is_bitmap(const unsigned long *bitmap)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Special-case data structure for "single bit set only" constant CPU masks.
|
|
|
|
*
|
|
|
|
* We pre-generate all the 64 (or 32) possible bit positions, with enough
|
|
|
|
* padding to the left and the right, and return the constant pointer
|
|
|
|
* appropriately offset.
|
|
|
|
*/
|
|
|
|
extern const unsigned long
|
|
|
|
cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
|
|
|
|
|
|
|
|
static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
|
|
|
|
{
|
|
|
|
const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
|
|
|
|
p -= cpu / BITS_PER_LONG;
|
|
|
|
return to_cpumask(p);
|
|
|
|
}
|
|
|
|
|
|
|
|
#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
|
|
|
|
|
|
|
|
#if NR_CPUS <= BITS_PER_LONG
|
|
|
|
#define CPU_BITS_ALL \
|
|
|
|
{ \
|
2015-03-05 08:19:19 +08:00
|
|
|
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
|
2009-09-24 23:34:53 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#else /* NR_CPUS > BITS_PER_LONG */
|
|
|
|
|
|
|
|
#define CPU_BITS_ALL \
|
|
|
|
{ \
|
|
|
|
[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
|
2015-03-05 08:19:19 +08:00
|
|
|
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
|
2009-09-24 23:34:53 +08:00
|
|
|
}
|
|
|
|
#endif /* NR_CPUS > BITS_PER_LONG */
|
|
|
|
|
2014-09-30 21:48:22 +08:00
|
|
|
/**
|
|
|
|
* cpumap_print_to_pagebuf - copies the cpumask into the buffer either
|
|
|
|
* as comma-separated list of cpus or hex values of cpumask
|
|
|
|
* @list: indicates whether the cpumap must be list
|
|
|
|
* @mask: the cpumask to copy
|
|
|
|
* @buf: the buffer to copy into
|
|
|
|
*
|
|
|
|
* Returns the length of the (null-terminated) @buf string, zero if
|
|
|
|
* nothing is copied.
|
|
|
|
*/
|
|
|
|
static inline ssize_t
|
|
|
|
cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
|
|
|
|
{
|
|
|
|
return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
|
cpumask: always use nr_cpu_ids in formatting and parsing functions
bitmap implements two variants of scnprintf functions to format a bitmap
into a string and cpumask and nodemask wrap them to provide equivalent
interfaces. The scnprintf family of functions require a string buffer as
an output target which complicates code paths which just want to print out
the mask through printk for informational or debug purposes as they have
to worry about how large the buffer should be and whether it's too large
to allocate on stack.
Neither cpumask or nodemask provides a guildeline on how large the target
buffer should be forcing users come up with their own solutions - some
allocate an arbitrarily sized buffer which is small enough to allocate on
stack but may be too short in corner cases, other come up with a custom
upper limit calculation considering the output format, some allocate the
buffer dynamically while one resorted to using lock to synchronize access
to a static buffer.
This is an artificial problem which is being solved repeatedly for no
benefit. In a lot of cases, the output area already exists and can be
targeted directly making the intermediate buffer unnecessary. This
patchset teaches printf family of functions how to format bitmaps and
replace the dedicated formatting functions with it.
Pointer formatting is extended to cover bitmap formatting. It uses the
field width for the number of bits instead of precision. The format used
is '%*pb[l]', with the optional trailing 'l' specifying list format
instead of hex masks. For more details, please see 0002.
This patch (of 31):
Currently, the formatting and parsing functions in cpumask.h use
nr_cpumask_bits like other cpumask functions; however, nr_cpumask_bits
is either NR_CPUS or nr_cpu_ids depending on CONFIG_CPUMASK_OFFSTACK.
This leads to inconsistent behaviors.
With CONFIG_NR_CPUS=512 and !CONFIG_CPUMASK_OFFSTACK
# cat /sys/devices/virtual/net/lo/queues/rx-0/rps_cpus
00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000
# cat /proc/self/status | grep Cpus_allowed:
Cpus_allowed: f
With CONFIG_NR_CPUS=1024 and CONFIG_CPUMASK_OFFSTACK (fedora default)
# cat /sys/devices/virtual/net/lo/queues/rx-0/rps_cpus
0
# cat /proc/self/status | grep Cpus_allowed:
Cpus_allowed: f
Note that /proc/self/status is always using nr_cpu_ids regardless of
config. This is because seq cpumask formattings functions always use
nr_cpu_ids.
Given that the same output fields may switch between the two forms,
converging on nr_cpu_ids always isn't too likely to surprise userland.
This patch updates the formatting and parsing functions in cpumask.h
to always use nr_cpu_ids. There's no point in dealing with CPUs which
aren't even possible on the machine.
Signed-off-by: Tejun Heo <tj@kernel.org>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
Cc: "John W. Linville" <linville@tuxdriver.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: Chris Zankel <chris@zankel.net>
Cc: Christoph Lameter <cl@linux.com>
Cc: Dmitry Torokhov <dmitry.torokhov@gmail.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Li Zefan <lizefan@huawei.com>
Cc: Max Filippov <jcmvbkbc@gmail.com>
Cc: Mike Travis <travis@sgi.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Russell King <linux@arm.linux.org.uk>
Acked-by: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steffen Klassert <steffen.klassert@secunet.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tony Luck <tony.luck@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-14 06:36:50 +08:00
|
|
|
nr_cpu_ids);
|
2014-09-30 21:48:22 +08:00
|
|
|
}
|
|
|
|
|
2009-09-24 23:34:53 +08:00
|
|
|
#if NR_CPUS <= BITS_PER_LONG
|
|
|
|
#define CPU_MASK_ALL \
|
|
|
|
(cpumask_t) { { \
|
2015-03-05 08:19:19 +08:00
|
|
|
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
|
2009-09-24 23:34:53 +08:00
|
|
|
} }
|
|
|
|
#else
|
|
|
|
#define CPU_MASK_ALL \
|
|
|
|
(cpumask_t) { { \
|
|
|
|
[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
|
2015-03-05 08:19:19 +08:00
|
|
|
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
|
2009-09-24 23:34:53 +08:00
|
|
|
} }
|
2015-03-05 08:19:19 +08:00
|
|
|
#endif /* NR_CPUS > BITS_PER_LONG */
|
2009-09-24 23:34:53 +08:00
|
|
|
|
|
|
|
#define CPU_MASK_NONE \
|
|
|
|
(cpumask_t) { { \
|
|
|
|
[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
|
|
|
|
} }
|
|
|
|
|
2015-04-16 11:03:51 +08:00
|
|
|
#define CPU_MASK_CPU0 \
|
|
|
|
(cpumask_t) { { \
|
|
|
|
[0] = 1UL \
|
|
|
|
} }
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif /* __LINUX_CPUMASK_H */
|