lk/linux-kernel-test
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'tj-percpu' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/misc into core/percpu

Browse Source 
Conflicts:
	arch/x86/kernel/setup_percpu.c

Semantic conflict:

	arch/x86/kernel/cpu/common.c

Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Ingo Molnar
2009-01-27 12:03:24 +01:00
parent 3ddeb51d9c cf3997f507
commit 4369f1fb7c
13 changed files with 326 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
arch/x86/Kconfig
View File

@@ -133,7 +133,7 @@ config ARCH_HAS_CACHE_LINE_SIZE
def_bool y def_bool y
config HAVE_SETUP_PER_CPU_AREA config HAVE_SETUP_PER_CPU_AREA
def_bool X86_64_SMP || (X86_SMP && !X86_VOYAGER) def_bool y
config HAVE_CPUMASK_OF_CPU_MAP config HAVE_CPUMASK_OF_CPU_MAP
def_bool X86_64_SMP def_bool X86_64_SMP

4
arch/x86/include/asm/cpumask.h
View File

@@ -10,6 +10,8 @@ extern cpumask_var_t cpu_callout_mask;
extern cpumask_var_t cpu_initialized_mask; extern cpumask_var_t cpu_initialized_mask;
extern cpumask_var_t cpu_sibling_setup_mask; extern cpumask_var_t cpu_sibling_setup_mask;
extern void setup_cpu_local_masks(void);
#else /* CONFIG_X86_32 */ #else /* CONFIG_X86_32 */
extern cpumask_t cpu_callin_map; extern cpumask_t cpu_callin_map;
@@ -22,6 +24,8 @@ extern cpumask_t cpu_sibling_setup_map;
#define cpu_initialized_mask ((struct cpumask *)&cpu_initialized) #define cpu_initialized_mask ((struct cpumask *)&cpu_initialized)
#define cpu_sibling_setup_mask ((struct cpumask *)&cpu_sibling_setup_map) #define cpu_sibling_setup_mask ((struct cpumask *)&cpu_sibling_setup_map)
static inline void setup_cpu_local_masks(void) { }
#endif /* CONFIG_X86_32 */ #endif /* CONFIG_X86_32 */
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */

9
arch/x86/include/asm/processor.h
View File

@@ -394,14 +394,6 @@ union irq_stack_union {
DECLARE_PER_CPU(union irq_stack_union, irq_stack_union); DECLARE_PER_CPU(union irq_stack_union, irq_stack_union);
DECLARE_PER_CPU(char *, irq_stack_ptr); DECLARE_PER_CPU(char *, irq_stack_ptr);
static inline void load_gs_base(int cpu)
{
/* Memory clobbers used to order pda/percpu accesses */
mb();
wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
mb();
}
#endif #endif
extern void print_cpu_info(struct cpuinfo_x86 *); extern void print_cpu_info(struct cpuinfo_x86 *);
@@ -778,7 +770,6 @@ extern struct desc_ptr early_gdt_descr;
extern void cpu_set_gdt(int); extern void cpu_set_gdt(int);
extern void switch_to_new_gdt(void); extern void switch_to_new_gdt(void);
extern void cpu_init(void); extern void cpu_init(void);
extern void init_gdt(int cpu);
static inline unsigned long get_debugctlmsr(void) static inline unsigned long get_debugctlmsr(void)
{ {

6
arch/x86/include/asm/topology.h
View File

@@ -74,6 +74,8 @@ static inline const struct cpumask *cpumask_of_node(int node)
return &node_to_cpumask_map[node]; return &node_to_cpumask_map[node];
} }
static inline void setup_node_to_cpumask_map(void) { }
#else /* CONFIG_X86_64 */ #else /* CONFIG_X86_64 */
/* Mappings between node number and cpus on that node. */ /* Mappings between node number and cpus on that node. */
@@ -120,6 +122,8 @@ static inline cpumask_t node_to_cpumask(int node)
#endif /* !CONFIG_DEBUG_PER_CPU_MAPS */ #endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
extern void setup_node_to_cpumask_map(void);
/* /*
* Replace default node_to_cpumask_ptr with optimized version * Replace default node_to_cpumask_ptr with optimized version
* Deprecated: use "const struct cpumask *mask = cpumask_of_node(node)" * Deprecated: use "const struct cpumask *mask = cpumask_of_node(node)"
@@ -218,6 +222,8 @@ static inline int node_to_first_cpu(int node)
return first_cpu(cpu_online_map); return first_cpu(cpu_online_map);
} }
static inline void setup_node_to_cpumask_map(void) { }
/* /*
* Replace default node_to_cpumask_ptr with optimized version * Replace default node_to_cpumask_ptr with optimized version
* Deprecated: use "const struct cpumask *mask = cpumask_of_node(node)" * Deprecated: use "const struct cpumask *mask = cpumask_of_node(node)"

6
arch/x86/kernel/Makefile
View File

@@ -28,7 +28,7 @@ CFLAGS_paravirt.o := $(nostackp)
obj-y := process_$(BITS).o signal.o entry_$(BITS).o obj-y := process_$(BITS).o signal.o entry_$(BITS).o
obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
obj-y += time_$(BITS).o ioport.o ldt.o dumpstack.o obj-y += time_$(BITS).o ioport.o ldt.o dumpstack.o
obj-y += setup.o i8259.o irqinit_$(BITS).o setup_percpu.o obj-y += setup.o i8259.o irqinit_$(BITS).o
obj-$(CONFIG_X86_VISWS) += visws_quirks.o obj-$(CONFIG_X86_VISWS) += visws_quirks.o
obj-$(CONFIG_X86_32) += probe_roms_32.o obj-$(CONFIG_X86_32) += probe_roms_32.o
obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o
@@ -59,8 +59,8 @@ apm-y := apm_32.o
obj-$(CONFIG_APM) += apm.o obj-$(CONFIG_APM) += apm.o
obj-$(CONFIG_X86_SMP) += smp.o obj-$(CONFIG_X86_SMP) += smp.o
obj-$(CONFIG_X86_SMP) += smpboot.o tsc_sync.o ipi.o obj-$(CONFIG_X86_SMP) += smpboot.o tsc_sync.o ipi.o
obj-$(CONFIG_X86_32_SMP) += smpcommon.o obj-$(CONFIG_SMP) += setup_percpu.o
obj-$(CONFIG_X86_64_SMP) += tsc_sync.o smpcommon.o obj-$(CONFIG_X86_64_SMP) += tsc_sync.o
obj-$(CONFIG_X86_TRAMPOLINE) += trampoline_$(BITS).o obj-$(CONFIG_X86_TRAMPOLINE) += trampoline_$(BITS).o
obj-$(CONFIG_X86_MPPARSE) += mpparse.o obj-$(CONFIG_X86_MPPARSE) += mpparse.o
obj-$(CONFIG_X86_LOCAL_APIC) += apic.o nmi.o obj-$(CONFIG_X86_LOCAL_APIC) += apic.o nmi.o

18
arch/x86/kernel/apic.c
View File

@@ -60,6 +60,24 @@
# error SPURIOUS_APIC_VECTOR definition error # error SPURIOUS_APIC_VECTOR definition error
#endif #endif
unsigned int num_processors;
unsigned disabled_cpus __cpuinitdata;
/* Processor that is doing the boot up */
unsigned int boot_cpu_physical_apicid = -1U;
EXPORT_SYMBOL(boot_cpu_physical_apicid);
unsigned int max_physical_apicid;
/* Bitmask of physically existing CPUs */
physid_mask_t phys_cpu_present_map;
/*
* Map cpu index to physical APIC ID
*/
DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
/* /*
* Knob to control our willingness to enable the local APIC. * Knob to control our willingness to enable the local APIC.

24
arch/x86/kernel/cpu/common.c
View File

@@ -52,6 +52,15 @@ cpumask_var_t cpu_initialized_mask;
/* representing cpus for which sibling maps can be computed */ /* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask; cpumask_var_t cpu_sibling_setup_mask;
/* correctly size the local cpu masks */
void __init setup_cpu_local_masks(void)
{
alloc_bootmem_cpumask_var(&cpu_initialized_mask);
alloc_bootmem_cpumask_var(&cpu_callin_mask);
alloc_bootmem_cpumask_var(&cpu_callout_mask);
alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}
#else /* CONFIG_X86_32 */ #else /* CONFIG_X86_32 */
cpumask_t cpu_callin_map; cpumask_t cpu_callin_map;
@@ -249,12 +258,17 @@ __u32 cleared_cpu_caps[NCAPINTS] __cpuinitdata;
void switch_to_new_gdt(void) void switch_to_new_gdt(void)
{ {
struct desc_ptr gdt_descr; struct desc_ptr gdt_descr;
int cpu = smp_processor_id();
gdt_descr.address = (long)get_cpu_gdt_table(smp_processor_id()); gdt_descr.address = (long)get_cpu_gdt_table(cpu);
gdt_descr.size = GDT_SIZE - 1; gdt_descr.size = GDT_SIZE - 1;
load_gdt(&gdt_descr); load_gdt(&gdt_descr);
/* Reload the per-cpu base */
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
asm("mov %0, %%fs" : : "r" (__KERNEL_PERCPU) : "memory"); loadsegment(fs, __KERNEL_PERCPU);
#else
loadsegment(gs, 0);
wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
#endif #endif
} }
@@ -959,10 +973,6 @@ void __cpuinit cpu_init(void)
struct task_struct *me; struct task_struct *me;
int i; int i;
loadsegment(fs, 0);
loadsegment(gs, 0);
load_gs_base(cpu);
#ifdef CONFIG_NUMA #ifdef CONFIG_NUMA
if (cpu != 0 && percpu_read(node_number) == 0 && if (cpu != 0 && percpu_read(node_number) == 0 &&
cpu_to_node(cpu) != NUMA_NO_NODE) cpu_to_node(cpu) != NUMA_NO_NODE)
@@ -984,6 +994,8 @@ void __cpuinit cpu_init(void)
*/ */
switch_to_new_gdt(); switch_to_new_gdt();
loadsegment(fs, 0);
load_idt((const struct desc_ptr *)&idt_descr); load_idt((const struct desc_ptr *)&idt_descr);
memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8); memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);

384
arch/x86/kernel/setup_percpu.c
View File

@@ -15,6 +15,7 @@
#include <asm/highmem.h> #include <asm/highmem.h>
#include <asm/proto.h> #include <asm/proto.h>
#include <asm/cpumask.h> #include <asm/cpumask.h>
#include <asm/cpu.h>
#ifdef CONFIG_DEBUG_PER_CPU_MAPS #ifdef CONFIG_DEBUG_PER_CPU_MAPS
# define DBG(x...) printk(KERN_DEBUG x) # define DBG(x...) printk(KERN_DEBUG x)
@@ -22,118 +23,36 @@
# define DBG(x...) # define DBG(x...)
#endif #endif
/*
* Could be inside CONFIG_HAVE_SETUP_PER_CPU_AREA with other stuff but
* voyager wants cpu_number too.
*/
#ifdef CONFIG_SMP
DEFINE_PER_CPU(int, cpu_number); DEFINE_PER_CPU(int, cpu_number);
EXPORT_PER_CPU_SYMBOL(cpu_number); EXPORT_PER_CPU_SYMBOL(cpu_number);
#endif
#ifdef CONFIG_X86_LOCAL_APIC
unsigned int num_processors;
unsigned disabled_cpus __cpuinitdata;
/* Processor that is doing the boot up */
unsigned int boot_cpu_physical_apicid = -1U;
EXPORT_SYMBOL(boot_cpu_physical_apicid);
unsigned int max_physical_apicid;
/* Bitmask of physically existing CPUs */
physid_mask_t phys_cpu_present_map;
#endif
/*
* Map cpu index to physical APIC ID
*/
DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
#define X86_64_NUMA 1 /* (used later) */
DEFINE_PER_CPU(int, node_number) = 0;
EXPORT_PER_CPU_SYMBOL(node_number);
/*
* Map cpu index to node index
*/
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
/*
* Which logical CPUs are on which nodes
*/
cpumask_t *node_to_cpumask_map;
EXPORT_SYMBOL(node_to_cpumask_map);
/*
* Setup node_to_cpumask_map
*/
static void __init setup_node_to_cpumask_map(void);
#ifdef CONFIG_X86_64
#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
#else #else
static inline void setup_node_to_cpumask_map(void) { } #define BOOT_PERCPU_OFFSET 0
#endif #endif
#ifdef CONFIG_X86_64 DEFINE_PER_CPU(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
EXPORT_PER_CPU_SYMBOL(this_cpu_off);
/* correctly size the local cpu masks */
static void __init setup_cpu_local_masks(void)
{
alloc_bootmem_cpumask_var(&cpu_initialized_mask);
alloc_bootmem_cpumask_var(&cpu_callin_mask);
alloc_bootmem_cpumask_var(&cpu_callout_mask);
alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}
#else /* CONFIG_X86_32 */
static inline void setup_cpu_local_masks(void)
{
}
#endif /* CONFIG_X86_32 */
#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
/*
* Copy data used in early init routines from the initial arrays to the
* per cpu data areas. These arrays then become expendable and the
* *_early_ptr's are zeroed indicating that the static arrays are gone.
*/
static void __init setup_per_cpu_maps(void)
{
int cpu;
for_each_possible_cpu(cpu) {
per_cpu(x86_cpu_to_apicid, cpu) =
early_per_cpu_map(x86_cpu_to_apicid, cpu);
per_cpu(x86_bios_cpu_apicid, cpu) =
early_per_cpu_map(x86_bios_cpu_apicid, cpu);
#ifdef X86_64_NUMA
per_cpu(x86_cpu_to_node_map, cpu) =
early_per_cpu_map(x86_cpu_to_node_map, cpu);
#endif
}
/* indicate the early static arrays will soon be gone */
early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
#ifdef X86_64_NUMA
early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
#endif
}
#ifdef CONFIG_X86_64
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = { unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
[0] = (unsigned long)__per_cpu_load, [0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
}; };
#else
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
#endif
EXPORT_SYMBOL(__per_cpu_offset); EXPORT_SYMBOL(__per_cpu_offset);
static inline void setup_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
struct desc_struct gdt;
pack_descriptor(&gdt, per_cpu_offset(cpu), 0xFFFFF,
0x2 | DESCTYPE_S, 0x8);
gdt.s = 1;
write_gdt_entry(get_cpu_gdt_table(cpu),
GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
#endif
}
/* /*
* Great future plan: * Great future plan:
* Declare PDA itself and support (irqstack,tss,pgd) as per cpu data. * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
@@ -141,15 +60,12 @@ EXPORT_SYMBOL(__per_cpu_offset);
*/ */
void __init setup_per_cpu_areas(void) void __init setup_per_cpu_areas(void)
{ {
ssize_t size, old_size; ssize_t size;
char *ptr; char *ptr;
int cpu; int cpu;
unsigned long align = 1;
/* Copy section for each CPU (we discard the original) */ /* Copy section for each CPU (we discard the original) */
old_size = PERCPU_ENOUGH_ROOM; size = roundup(PERCPU_ENOUGH_ROOM, PAGE_SIZE);
align = max_t(unsigned long, PAGE_SIZE, align);
size = roundup(old_size, align);
pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n", pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n",
NR_CPUS, nr_cpumask_bits, nr_cpu_ids, nr_node_ids); NR_CPUS, nr_cpumask_bits, nr_cpu_ids, nr_node_ids);
@@ -158,20 +74,17 @@ void __init setup_per_cpu_areas(void)
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
#ifndef CONFIG_NEED_MULTIPLE_NODES #ifndef CONFIG_NEED_MULTIPLE_NODES
ptr = __alloc_bootmem(size, align, ptr = alloc_bootmem_pages(size);
__pa(MAX_DMA_ADDRESS));
#else #else
int node = early_cpu_to_node(cpu); int node = early_cpu_to_node(cpu);
if (!node_online(node) || !NODE_DATA(node)) { if (!node_online(node) || !NODE_DATA(node)) {
ptr = __alloc_bootmem(size, align, ptr = alloc_bootmem_pages(size);
__pa(MAX_DMA_ADDRESS));
pr_info("cpu %d has no node %d or node-local memory\n", pr_info("cpu %d has no node %d or node-local memory\n",
cpu, node); cpu, node);
pr_debug("per cpu data for cpu%d at %016lx\n", pr_debug("per cpu data for cpu%d at %016lx\n",
cpu, __pa(ptr)); cpu, __pa(ptr));
} else { } else {
ptr = __alloc_bootmem_node(NODE_DATA(node), size, align, ptr = alloc_bootmem_pages_node(NODE_DATA(node), size);
__pa(MAX_DMA_ADDRESS));
pr_debug("per cpu data for cpu%d on node%d at %016lx\n", pr_debug("per cpu data for cpu%d on node%d at %016lx\n",
cpu, node, __pa(ptr)); cpu, node, __pa(ptr));
} }
@@ -181,22 +94,47 @@ void __init setup_per_cpu_areas(void)
per_cpu_offset(cpu) = ptr - __per_cpu_start; per_cpu_offset(cpu) = ptr - __per_cpu_start;
per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu); per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
per_cpu(cpu_number, cpu) = cpu; per_cpu(cpu_number, cpu) = cpu;
setup_percpu_segment(cpu);
/*
* Copy data used in early init routines from the
* initial arrays to the per cpu data areas. These
* arrays then become expendable and the *_early_ptr's
* are zeroed indicating that the static arrays are
* gone.
*/
#ifdef CONFIG_X86_LOCAL_APIC
per_cpu(x86_cpu_to_apicid, cpu) =
early_per_cpu_map(x86_cpu_to_apicid, cpu);
per_cpu(x86_bios_cpu_apicid, cpu) =
early_per_cpu_map(x86_bios_cpu_apicid, cpu);
#endif
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack_ptr, cpu) =
per_cpu(irq_stack_union.irq_stack, cpu) + IRQ_STACK_SIZE - 64; per_cpu(irq_stack_union.irq_stack, cpu) +
/* IRQ_STACK_SIZE - 64;
* Up to this point, CPU0 has been using .data.init #ifdef CONFIG_NUMA
* area. Reload %gs offset for CPU0. per_cpu(x86_cpu_to_node_map, cpu) =
*/ early_per_cpu_map(x86_cpu_to_node_map, cpu);
if (cpu == 0)
load_gs_base(cpu);
#endif #endif
#endif
/*
* Up to this point, the boot CPU has been using .data.init
* area. Reload any changed state for the boot CPU.
*/
if (cpu == boot_cpu_id)
switch_to_new_gdt();
DBG("PERCPU: cpu %4d %p\n", cpu, ptr); DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
} }
/* Setup percpu data maps */ /* indicate the early static arrays will soon be gone */
setup_per_cpu_maps(); #ifdef CONFIG_X86_LOCAL_APIC
early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
#endif
#if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
#endif
/* Setup node to cpumask map */ /* Setup node to cpumask map */
setup_node_to_cpumask_map(); setup_node_to_cpumask_map();
@@ -204,207 +142,3 @@ void __init setup_per_cpu_areas(void)
/* Setup cpu initialized, callin, callout masks */ /* Setup cpu initialized, callin, callout masks */
setup_cpu_local_masks(); setup_cpu_local_masks();
} }
#endif
#ifdef X86_64_NUMA
/*
* Allocate node_to_cpumask_map based on number of available nodes
* Requires node_possible_map to be valid.
*
* Note: node_to_cpumask() is not valid until after this is done.
* (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
*/
static void __init setup_node_to_cpumask_map(void)
{
unsigned int node, num = 0;
cpumask_t *map;
/* setup nr_node_ids if not done yet */
if (nr_node_ids == MAX_NUMNODES) {
for_each_node_mask(node, node_possible_map)
num = node;
nr_node_ids = num + 1;
}
/* allocate the map */
map = alloc_bootmem_low(nr_node_ids * sizeof(cpumask_t));
DBG("node_to_cpumask_map at %p for %d nodes\n", map, nr_node_ids);
pr_debug("Node to cpumask map at %p for %d nodes\n",
map, nr_node_ids);
/* node_to_cpumask() will now work */
node_to_cpumask_map = map;
}
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
/* early setting, no percpu area yet */
if (cpu_to_node_map) {
cpu_to_node_map[cpu] = node;
return;
}
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
if (cpu >= nr_cpu_ids || !per_cpu_offset(cpu)) {
printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
dump_stack();
return;
}
#endif
per_cpu(x86_cpu_to_node_map, cpu) = node;
if (node != NUMA_NO_NODE)
per_cpu(node_number, cpu) = node;
}
void __cpuinit numa_clear_node(int cpu)
{
numa_set_node(cpu, NUMA_NO_NODE);
}
#ifndef CONFIG_DEBUG_PER_CPU_MAPS
void __cpuinit numa_add_cpu(int cpu)
{
cpu_set(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
}
void __cpuinit numa_remove_cpu(int cpu)
{
cpu_clear(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
}
#else /* CONFIG_DEBUG_PER_CPU_MAPS */
/*
* --------- debug versions of the numa functions ---------
*/
static void __cpuinit numa_set_cpumask(int cpu, int enable)
{
int node = early_cpu_to_node(cpu);
cpumask_t *mask;
char buf[64];
if (node_to_cpumask_map == NULL) {
printk(KERN_ERR "node_to_cpumask_map NULL\n");
dump_stack();
return;
}
mask = &node_to_cpumask_map[node];
if (enable)
cpu_set(cpu, *mask);
else
cpu_clear(cpu, *mask);
cpulist_scnprintf(buf, sizeof(buf), mask);
printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
enable ? "numa_add_cpu" : "numa_remove_cpu", cpu, node, buf);
}
void __cpuinit numa_add_cpu(int cpu)
{
numa_set_cpumask(cpu, 1);
}
void __cpuinit numa_remove_cpu(int cpu)
{
numa_set_cpumask(cpu, 0);
}
int cpu_to_node(int cpu)
{
if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
printk(KERN_WARNING
"cpu_to_node(%d): usage too early!\n", cpu);
dump_stack();
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
}
return per_cpu(x86_cpu_to_node_map, cpu);
}
EXPORT_SYMBOL(cpu_to_node);
/*
* Same function as cpu_to_node() but used if called before the
* per_cpu areas are setup.
*/
int early_cpu_to_node(int cpu)
{
if (early_per_cpu_ptr(x86_cpu_to_node_map))
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
if (!per_cpu_offset(cpu)) {
printk(KERN_WARNING
"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
dump_stack();
return NUMA_NO_NODE;
}
return per_cpu(x86_cpu_to_node_map, cpu);
}
/* empty cpumask */
static const cpumask_t cpu_mask_none;
/*
* Returns a pointer to the bitmask of CPUs on Node 'node'.
*/
const cpumask_t *cpumask_of_node(int node)
{
if (node_to_cpumask_map == NULL) {
printk(KERN_WARNING
"cpumask_of_node(%d): no node_to_cpumask_map!\n",
node);
dump_stack();
return (const cpumask_t *)&cpu_online_map;
}
if (node >= nr_node_ids) {
printk(KERN_WARNING
"cpumask_of_node(%d): node > nr_node_ids(%d)\n",
node, nr_node_ids);
dump_stack();
return &cpu_mask_none;
}
return &node_to_cpumask_map[node];
}
EXPORT_SYMBOL(cpumask_of_node);
/*
* Returns a bitmask of CPUs on Node 'node'.
*
* Side note: this function creates the returned cpumask on the stack
* so with a high NR_CPUS count, excessive stack space is used. The
* node_to_cpumask_ptr function should be used whenever possible.
*/
cpumask_t node_to_cpumask(int node)
{
if (node_to_cpumask_map == NULL) {
printk(KERN_WARNING
"node_to_cpumask(%d): no node_to_cpumask_map!\n", node);
dump_stack();
return cpu_online_map;
}
if (node >= nr_node_ids) {
printk(KERN_WARNING
"node_to_cpumask(%d): node > nr_node_ids(%d)\n",
node, nr_node_ids);
dump_stack();
return cpu_mask_none;
}
return node_to_cpumask_map[node];
}
EXPORT_SYMBOL(node_to_cpumask);
/*
* --------- end of debug versions of the numa functions ---------
*/
#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
#endif /* X86_64_NUMA */

4
arch/x86/kernel/smpboot.c
View File

@@ -793,7 +793,6 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
do_rest: do_rest:
per_cpu(current_task, cpu) = c_idle.idle; per_cpu(current_task, cpu) = c_idle.idle;
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
init_gdt(cpu);
/* Stack for startup_32 can be just as for start_secondary onwards */ /* Stack for startup_32 can be just as for start_secondary onwards */
irq_ctx_init(cpu); irq_ctx_init(cpu);
#else #else
@@ -1186,9 +1185,6 @@ out:
void __init native_smp_prepare_boot_cpu(void) void __init native_smp_prepare_boot_cpu(void)
{ {
int me = smp_processor_id(); int me = smp_processor_id();
#ifdef CONFIG_X86_32
init_gdt(me);
#endif
switch_to_new_gdt(); switch_to_new_gdt();
/* already set me in cpu_online_mask in boot_cpu_init() */ /* already set me in cpu_online_mask in boot_cpu_init() */
cpumask_set_cpu(me, cpu_callout_mask); cpumask_set_cpu(me, cpu_callout_mask);

32
arch/x86/kernel/smpcommon.c
View File

@@ -1,32 +0,0 @@
/*
* SMP stuff which is common to all sub-architectures.
*/
#include <linux/module.h>
#include <asm/smp.h>
#include <asm/sections.h>
#ifdef CONFIG_X86_64
DEFINE_PER_CPU(unsigned long, this_cpu_off) = (unsigned long)__per_cpu_load;
#else
DEFINE_PER_CPU(unsigned long, this_cpu_off);
#endif
EXPORT_PER_CPU_SYMBOL(this_cpu_off);
#ifdef CONFIG_X86_32
/*
* Initialize the CPU's GDT. This is either the boot CPU doing itself
* (still using the master per-cpu area), or a CPU doing it for a
* secondary which will soon come up.
*/
__cpuinit void init_gdt(int cpu)
{
struct desc_struct gdt;
pack_descriptor(&gdt, __per_cpu_offset[cpu], 0xFFFFF,
0x2 | DESCTYPE_S, 0x8);
gdt.s = 1;
write_gdt_entry(get_cpu_gdt_table(cpu),
GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
}
#endif

5
arch/x86/mach-voyager/voyager_smp.c
View File

@@ -530,8 +530,6 @@ static void __init do_boot_cpu(__u8 cpu)
/* init_tasks (in sched.c) is indexed logically */ /* init_tasks (in sched.c) is indexed logically */
stack_start.sp = (void *)idle->thread.sp; stack_start.sp = (void *)idle->thread.sp;
init_gdt(cpu);
per_cpu(this_cpu_off, cpu) = __per_cpu_offset[cpu];
per_cpu(current_task, cpu) = idle; per_cpu(current_task, cpu) = idle;
early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu); early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
irq_ctx_init(cpu); irq_ctx_init(cpu);
@@ -1748,8 +1746,6 @@ static void __init voyager_smp_prepare_cpus(unsigned int max_cpus)
static void __cpuinit voyager_smp_prepare_boot_cpu(void) static void __cpuinit voyager_smp_prepare_boot_cpu(void)
{ {
init_gdt(smp_processor_id());
per_cpu(this_cpu_off, cpu) = __per_cpu_offset[cpu];
switch_to_new_gdt(); switch_to_new_gdt();
cpu_set(smp_processor_id(), cpu_online_map); cpu_set(smp_processor_id(), cpu_online_map);
@@ -1782,7 +1778,6 @@ static void __init voyager_smp_cpus_done(unsigned int max_cpus)
void __init smp_setup_processor_id(void) void __init smp_setup_processor_id(void)
{ {
current_thread_info()->cpu = hard_smp_processor_id(); current_thread_info()->cpu = hard_smp_processor_id();
percpu_write(cpu_number, hard_smp_processor_id());
} }
static void voyager_send_call_func(cpumask_t callmask) static void voyager_send_call_func(cpumask_t callmask)

217
arch/x86/mm/numa_64.c
View File

@@ -20,6 +20,12 @@
#include <asm/acpi.h> #include <asm/acpi.h>
#include <asm/k8.h> #include <asm/k8.h>
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
# define DBG(x...) printk(KERN_DEBUG x)
#else
# define DBG(x...)
#endif
struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
EXPORT_SYMBOL(node_data); EXPORT_SYMBOL(node_data);
@@ -33,6 +39,21 @@ int numa_off __initdata;
static unsigned long __initdata nodemap_addr; static unsigned long __initdata nodemap_addr;
static unsigned long __initdata nodemap_size; static unsigned long __initdata nodemap_size;
DEFINE_PER_CPU(int, node_number) = 0;
EXPORT_PER_CPU_SYMBOL(node_number);
/*
* Map cpu index to node index
*/
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
/*
* Which logical CPUs are on which nodes
*/
cpumask_t *node_to_cpumask_map;
EXPORT_SYMBOL(node_to_cpumask_map);
/* /*
* Given a shift value, try to populate memnodemap[] * Given a shift value, try to populate memnodemap[]
* Returns : * Returns :
@@ -640,3 +661,199 @@ void __init init_cpu_to_node(void)
#endif #endif
/*
* Allocate node_to_cpumask_map based on number of available nodes
* Requires node_possible_map to be valid.
*
* Note: node_to_cpumask() is not valid until after this is done.
* (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
*/
void __init setup_node_to_cpumask_map(void)
{
unsigned int node, num = 0;
cpumask_t *map;
/* setup nr_node_ids if not done yet */
if (nr_node_ids == MAX_NUMNODES) {
for_each_node_mask(node, node_possible_map)
num = node;
nr_node_ids = num + 1;
}
/* allocate the map */
map = alloc_bootmem_low(nr_node_ids * sizeof(cpumask_t));
DBG("node_to_cpumask_map at %p for %d nodes\n", map, nr_node_ids);
pr_debug("Node to cpumask map at %p for %d nodes\n",
map, nr_node_ids);
/* node_to_cpumask() will now work */
node_to_cpumask_map = map;
}
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
/* early setting, no percpu area yet */
if (cpu_to_node_map) {
cpu_to_node_map[cpu] = node;
return;
}
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
if (cpu >= nr_cpu_ids || !per_cpu_offset(cpu)) {
printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
dump_stack();
return;
}
#endif
per_cpu(x86_cpu_to_node_map, cpu) = node;
if (node != NUMA_NO_NODE)
per_cpu(node_number, cpu) = node;
}
void __cpuinit numa_clear_node(int cpu)
{
numa_set_node(cpu, NUMA_NO_NODE);
}
#ifndef CONFIG_DEBUG_PER_CPU_MAPS
void __cpuinit numa_add_cpu(int cpu)
{
cpu_set(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
}
void __cpuinit numa_remove_cpu(int cpu)
{
cpu_clear(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
}
#else /* CONFIG_DEBUG_PER_CPU_MAPS */
/*
* --------- debug versions of the numa functions ---------
*/
static void __cpuinit numa_set_cpumask(int cpu, int enable)
{
int node = early_cpu_to_node(cpu);
cpumask_t *mask;
char buf[64];
if (node_to_cpumask_map == NULL) {
printk(KERN_ERR "node_to_cpumask_map NULL\n");
dump_stack();
return;
}
mask = &node_to_cpumask_map[node];
if (enable)
cpu_set(cpu, *mask);
else
cpu_clear(cpu, *mask);
cpulist_scnprintf(buf, sizeof(buf), mask);
printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
enable ? "numa_add_cpu" : "numa_remove_cpu", cpu, node, buf);
}
void __cpuinit numa_add_cpu(int cpu)
{
numa_set_cpumask(cpu, 1);
}
void __cpuinit numa_remove_cpu(int cpu)
{
numa_set_cpumask(cpu, 0);
}
int cpu_to_node(int cpu)
{
if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
printk(KERN_WARNING
"cpu_to_node(%d): usage too early!\n", cpu);
dump_stack();
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
}
return per_cpu(x86_cpu_to_node_map, cpu);
}
EXPORT_SYMBOL(cpu_to_node);
/*
* Same function as cpu_to_node() but used if called before the
* per_cpu areas are setup.
*/
int early_cpu_to_node(int cpu)
{
if (early_per_cpu_ptr(x86_cpu_to_node_map))
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
if (!per_cpu_offset(cpu)) {
printk(KERN_WARNING
"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
dump_stack();
return NUMA_NO_NODE;
}
return per_cpu(x86_cpu_to_node_map, cpu);
}
/* empty cpumask */
static const cpumask_t cpu_mask_none;
/*
* Returns a pointer to the bitmask of CPUs on Node 'node'.
*/
const cpumask_t *cpumask_of_node(int node)
{
if (node_to_cpumask_map == NULL) {
printk(KERN_WARNING
"cpumask_of_node(%d): no node_to_cpumask_map!\n",
node);
dump_stack();
return (const cpumask_t *)&cpu_online_map;
}
if (node >= nr_node_ids) {
printk(KERN_WARNING
"cpumask_of_node(%d): node > nr_node_ids(%d)\n",
node, nr_node_ids);
dump_stack();
return &cpu_mask_none;
}
return &node_to_cpumask_map[node];
}
EXPORT_SYMBOL(cpumask_of_node);
/*
* Returns a bitmask of CPUs on Node 'node'.
*
* Side note: this function creates the returned cpumask on the stack
* so with a high NR_CPUS count, excessive stack space is used. The
* node_to_cpumask_ptr function should be used whenever possible.
*/
cpumask_t node_to_cpumask(int node)
{
if (node_to_cpumask_map == NULL) {
printk(KERN_WARNING
"node_to_cpumask(%d): no node_to_cpumask_map!\n", node);
dump_stack();
return cpu_online_map;
}
if (node >= nr_node_ids) {
printk(KERN_WARNING
"node_to_cpumask(%d): node > nr_node_ids(%d)\n",
node, nr_node_ids);
dump_stack();
return cpu_mask_none;
}
return node_to_cpumask_map[node];
}
EXPORT_SYMBOL(node_to_cpumask);
/*
* --------- end of debug versions of the numa functions ---------
*/
#endif /* CONFIG_DEBUG_PER_CPU_MAPS */

1
arch/x86/xen/smp.c
View File

@@ -281,7 +281,6 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)
per_cpu(current_task, cpu) = idle; per_cpu(current_task, cpu) = idle;
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
init_gdt(cpu);
irq_ctx_init(cpu); irq_ctx_init(cpu);
#else #else
clear_tsk_thread_flag(idle, TIF_FORK); clear_tsk_thread_flag(idle, TIF_FORK);