2008-10-23 13:26:29 +08:00
|
|
|
#ifndef _ASM_X86_SMP_H
|
|
|
|
#define _ASM_X86_SMP_H
|
2008-03-04 01:12:29 +08:00
|
|
|
#ifndef __ASSEMBLY__
|
2008-03-04 01:12:31 +08:00
|
|
|
#include <linux/cpumask.h>
|
2008-03-04 01:12:40 +08:00
|
|
|
#include <linux/init.h>
|
2008-03-20 01:25:18 +08:00
|
|
|
#include <asm/percpu.h>
|
2008-03-04 01:12:31 +08:00
|
|
|
|
2008-03-28 01:06:00 +08:00
|
|
|
/*
|
|
|
|
* We need the APIC definitions automatically as part of 'smp.h'
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_X86_LOCAL_APIC
|
|
|
|
# include <asm/mpspec.h>
|
|
|
|
# include <asm/apic.h>
|
|
|
|
# ifdef CONFIG_X86_IO_APIC
|
|
|
|
# include <asm/io_apic.h>
|
|
|
|
# endif
|
|
|
|
#endif
|
|
|
|
#include <asm/thread_info.h>
|
2009-01-10 14:50:24 +08:00
|
|
|
#include <asm/cpumask.h>
|
2008-03-28 01:06:00 +08:00
|
|
|
|
2008-03-04 01:12:31 +08:00
|
|
|
extern int smp_num_siblings;
|
|
|
|
extern unsigned int num_processors;
|
2008-03-04 01:12:29 +08:00
|
|
|
|
2009-03-13 12:19:50 +08:00
|
|
|
DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_map);
|
|
|
|
DECLARE_PER_CPU(cpumask_var_t, cpu_core_map);
|
2008-03-20 01:25:18 +08:00
|
|
|
DECLARE_PER_CPU(u16, cpu_llc_id);
|
2008-07-22 00:06:40 +08:00
|
|
|
DECLARE_PER_CPU(int, cpu_number);
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-13 03:21:12 +08:00
|
|
|
|
2009-01-04 21:18:03 +08:00
|
|
|
static inline struct cpumask *cpu_sibling_mask(int cpu)
|
|
|
|
{
|
2009-03-13 12:19:50 +08:00
|
|
|
return per_cpu(cpu_sibling_map, cpu);
|
2009-01-04 21:18:03 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct cpumask *cpu_core_mask(int cpu)
|
|
|
|
{
|
2009-03-13 12:19:50 +08:00
|
|
|
return per_cpu(cpu_core_map, cpu);
|
2009-01-04 21:18:03 +08:00
|
|
|
}
|
|
|
|
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-13 03:21:12 +08:00
|
|
|
DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid);
|
|
|
|
DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid);
|
2008-03-20 01:25:18 +08:00
|
|
|
|
2008-03-20 01:25:57 +08:00
|
|
|
/* Static state in head.S used to set up a CPU */
|
|
|
|
extern struct {
|
|
|
|
void *sp;
|
|
|
|
unsigned short ss;
|
|
|
|
} stack_start;
|
|
|
|
|
2008-03-04 01:12:32 +08:00
|
|
|
struct smp_ops {
|
|
|
|
void (*smp_prepare_boot_cpu)(void);
|
|
|
|
void (*smp_prepare_cpus)(unsigned max_cpus);
|
|
|
|
void (*smp_cpus_done)(unsigned max_cpus);
|
|
|
|
|
|
|
|
void (*smp_send_stop)(void);
|
|
|
|
void (*smp_send_reschedule)(int cpu);
|
2008-06-26 17:21:54 +08:00
|
|
|
|
2008-08-22 18:52:11 +08:00
|
|
|
int (*cpu_up)(unsigned cpu);
|
|
|
|
int (*cpu_disable)(void);
|
|
|
|
void (*cpu_die)(unsigned int cpu);
|
|
|
|
void (*play_dead)(void);
|
|
|
|
|
2008-12-17 09:33:59 +08:00
|
|
|
void (*send_call_func_ipi)(const struct cpumask *mask);
|
2008-06-26 17:21:54 +08:00
|
|
|
void (*send_call_func_single_ipi)(int cpu);
|
2008-03-04 01:12:32 +08:00
|
|
|
};
|
|
|
|
|
2008-03-04 01:12:59 +08:00
|
|
|
/* Globals due to paravirt */
|
|
|
|
extern void set_cpu_sibling_map(int cpu);
|
|
|
|
|
2008-03-04 01:12:33 +08:00
|
|
|
#ifdef CONFIG_SMP
|
2008-03-20 01:24:59 +08:00
|
|
|
#ifndef CONFIG_PARAVIRT
|
|
|
|
#define startup_ipi_hook(phys_apicid, start_eip, start_esp) do { } while (0)
|
|
|
|
#endif
|
2008-03-04 01:12:33 +08:00
|
|
|
extern struct smp_ops smp_ops;
|
2008-03-04 01:12:34 +08:00
|
|
|
|
2008-03-04 01:12:51 +08:00
|
|
|
static inline void smp_send_stop(void)
|
|
|
|
{
|
|
|
|
smp_ops.smp_send_stop();
|
|
|
|
}
|
|
|
|
|
2008-03-04 01:12:37 +08:00
|
|
|
static inline void smp_prepare_boot_cpu(void)
|
|
|
|
{
|
|
|
|
smp_ops.smp_prepare_boot_cpu();
|
|
|
|
}
|
|
|
|
|
2008-03-04 01:12:38 +08:00
|
|
|
static inline void smp_prepare_cpus(unsigned int max_cpus)
|
|
|
|
{
|
|
|
|
smp_ops.smp_prepare_cpus(max_cpus);
|
|
|
|
}
|
|
|
|
|
2008-03-04 01:12:39 +08:00
|
|
|
static inline void smp_cpus_done(unsigned int max_cpus)
|
|
|
|
{
|
|
|
|
smp_ops.smp_cpus_done(max_cpus);
|
|
|
|
}
|
|
|
|
|
2008-03-04 01:12:36 +08:00
|
|
|
static inline int __cpu_up(unsigned int cpu)
|
|
|
|
{
|
|
|
|
return smp_ops.cpu_up(cpu);
|
|
|
|
}
|
|
|
|
|
2008-08-22 18:52:11 +08:00
|
|
|
static inline int __cpu_disable(void)
|
|
|
|
{
|
|
|
|
return smp_ops.cpu_disable();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void __cpu_die(unsigned int cpu)
|
|
|
|
{
|
|
|
|
smp_ops.cpu_die(cpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void play_dead(void)
|
|
|
|
{
|
|
|
|
smp_ops.play_dead();
|
|
|
|
}
|
|
|
|
|
2008-03-04 01:12:34 +08:00
|
|
|
static inline void smp_send_reschedule(int cpu)
|
|
|
|
{
|
|
|
|
smp_ops.smp_send_reschedule(cpu);
|
|
|
|
}
|
2008-03-04 01:12:35 +08:00
|
|
|
|
2008-06-26 17:21:54 +08:00
|
|
|
static inline void arch_send_call_function_single_ipi(int cpu)
|
|
|
|
{
|
|
|
|
smp_ops.send_call_func_single_ipi(cpu);
|
|
|
|
}
|
|
|
|
|
2009-03-13 12:19:51 +08:00
|
|
|
#define arch_send_call_function_ipi_mask arch_send_call_function_ipi_mask
|
|
|
|
static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask)
|
2008-03-04 01:12:35 +08:00
|
|
|
{
|
2009-03-13 12:19:51 +08:00
|
|
|
smp_ops.send_call_func_ipi(mask);
|
2008-03-04 01:12:35 +08:00
|
|
|
}
|
2008-03-04 01:12:36 +08:00
|
|
|
|
2008-08-22 18:52:14 +08:00
|
|
|
void cpu_disable_common(void);
|
2008-03-04 01:12:37 +08:00
|
|
|
void native_smp_prepare_boot_cpu(void);
|
2008-03-04 01:12:38 +08:00
|
|
|
void native_smp_prepare_cpus(unsigned int max_cpus);
|
2008-03-04 01:12:39 +08:00
|
|
|
void native_smp_cpus_done(unsigned int max_cpus);
|
2008-03-04 01:12:36 +08:00
|
|
|
int native_cpu_up(unsigned int cpunum);
|
2008-08-22 18:52:11 +08:00
|
|
|
int native_cpu_disable(void);
|
|
|
|
void native_cpu_die(unsigned int cpu);
|
|
|
|
void native_play_dead(void);
|
2008-08-22 18:52:13 +08:00
|
|
|
void play_dead_common(void);
|
2008-08-22 18:52:11 +08:00
|
|
|
|
2008-12-17 09:33:59 +08:00
|
|
|
void native_send_call_func_ipi(const struct cpumask *mask);
|
2008-06-26 17:21:54 +08:00
|
|
|
void native_send_call_func_single_ipi(int cpu);
|
2008-03-04 01:12:40 +08:00
|
|
|
|
2008-03-20 01:25:05 +08:00
|
|
|
void smp_store_cpu_info(int id);
|
2008-03-20 01:25:58 +08:00
|
|
|
#define cpu_physical_id(cpu) per_cpu(x86_cpu_to_apicid, cpu)
|
2008-03-28 01:06:02 +08:00
|
|
|
|
|
|
|
/* We don't mark CPUs online until __cpu_up(), so we need another measure */
|
|
|
|
static inline int num_booting_cpus(void)
|
|
|
|
{
|
2009-01-04 21:18:03 +08:00
|
|
|
return cpumask_weight(cpu_callout_mask);
|
2008-03-28 01:06:02 +08:00
|
|
|
}
|
2008-09-30 06:29:42 +08:00
|
|
|
#endif /* CONFIG_SMP */
|
2008-03-28 01:06:02 +08:00
|
|
|
|
2008-04-05 03:41:44 +08:00
|
|
|
extern unsigned disabled_cpus __cpuinitdata;
|
|
|
|
|
2008-03-28 01:06:02 +08:00
|
|
|
#ifdef CONFIG_X86_32_SMP
|
|
|
|
/*
|
|
|
|
* This function is needed by all SMP systems. It must _always_ be valid
|
|
|
|
* from the initial startup. We map APIC_BASE very early in page_setup(),
|
|
|
|
* so this is correct in the x86 case.
|
|
|
|
*/
|
percpu: add optimized generic percpu accessors
It is an optimization and a cleanup, and adds the following new
generic percpu methods:
percpu_read()
percpu_write()
percpu_add()
percpu_sub()
percpu_and()
percpu_or()
percpu_xor()
and implements support for them on x86. (other architectures will fall
back to a default implementation)
The advantage is that for example to read a local percpu variable,
instead of this sequence:
return __get_cpu_var(var);
ffffffff8102ca2b: 48 8b 14 fd 80 09 74 mov -0x7e8bf680(,%rdi,8),%rdx
ffffffff8102ca32: 81
ffffffff8102ca33: 48 c7 c0 d8 59 00 00 mov $0x59d8,%rax
ffffffff8102ca3a: 48 8b 04 10 mov (%rax,%rdx,1),%rax
We can get a single instruction by using the optimized variants:
return percpu_read(var);
ffffffff8102ca3f: 65 48 8b 05 91 8f fd mov %gs:0x7efd8f91(%rip),%rax
I also cleaned up the x86-specific APIs and made the x86 code use
these new generic percpu primitives.
tj: * fixed generic percpu_sub() definition as Roel Kluin pointed out
* added percpu_and() for completeness's sake
* made generic percpu ops atomic against preemption
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Tejun Heo <tj@kernel.org>
2009-01-15 21:15:53 +08:00
|
|
|
#define raw_smp_processor_id() (percpu_read(cpu_number))
|
2008-03-28 01:06:02 +08:00
|
|
|
extern int safe_smp_processor_id(void);
|
|
|
|
|
|
|
|
#elif defined(CONFIG_X86_64_SMP)
|
2009-01-18 23:38:58 +08:00
|
|
|
#define raw_smp_processor_id() (percpu_read(cpu_number))
|
2008-03-28 01:06:02 +08:00
|
|
|
|
|
|
|
#define stack_smp_processor_id() \
|
|
|
|
({ \
|
|
|
|
struct thread_info *ti; \
|
|
|
|
__asm__("andq %%rsp,%0; ":"=r" (ti) : "0" (CURRENT_MASK)); \
|
|
|
|
ti->cpu; \
|
|
|
|
})
|
|
|
|
#define safe_smp_processor_id() smp_processor_id()
|
|
|
|
|
2008-03-04 01:12:33 +08:00
|
|
|
#endif
|
2008-03-04 01:12:32 +08:00
|
|
|
|
2008-03-28 01:05:58 +08:00
|
|
|
#ifdef CONFIG_X86_LOCAL_APIC
|
|
|
|
|
2008-07-11 02:16:49 +08:00
|
|
|
#ifndef CONFIG_X86_64
|
2008-03-28 01:05:58 +08:00
|
|
|
static inline int logical_smp_processor_id(void)
|
|
|
|
{
|
|
|
|
/* we don't want to mark this access volatile - bad code generation */
|
|
|
|
return GET_APIC_LOGICAL_ID(*(u32 *)(APIC_BASE + APIC_LDR));
|
|
|
|
}
|
|
|
|
|
2008-03-29 03:12:16 +08:00
|
|
|
#endif
|
|
|
|
|
2008-03-28 01:05:58 +08:00
|
|
|
extern int hard_smp_processor_id(void);
|
|
|
|
|
|
|
|
#else /* CONFIG_X86_LOCAL_APIC */
|
|
|
|
|
|
|
|
# ifndef CONFIG_SMP
|
|
|
|
# define hard_smp_processor_id() 0
|
|
|
|
# endif
|
|
|
|
|
|
|
|
#endif /* CONFIG_X86_LOCAL_APIC */
|
|
|
|
|
2008-03-04 01:12:29 +08:00
|
|
|
#endif /* __ASSEMBLY__ */
|
2008-10-23 13:26:29 +08:00
|
|
|
#endif /* _ASM_X86_SMP_H */
|