linux/init/main.c
Christoph Lameter e7c8d5c995 [PATCH] node local per-cpu-pages
This patch modifies the way pagesets in struct zone are managed.

Each zone has a per-cpu array of pagesets.  So any particular CPU has some
memory in each zone structure which belongs to itself.  Even if that CPU is
not local to that zone.

So the patch relocates the pagesets for each cpu to the node that is nearest
to the cpu instead of allocating the pagesets in the (possibly remote) target
zone.  This means that the operations to manage pages on remote zone can be
done with information available locally.

We play a macro trick so that non-NUMA pmachines avoid the additional
pointer chase on the page allocator fastpath.

AIM7 benchmark on a 32 CPU SGI Altix

w/o patches:
Tasks    jobs/min  jti  jobs/min/task      real       cpu
    1      484.68  100       484.6769     12.01      1.97   Fri Mar 25 11:01:42 2005
  100    27140.46   89       271.4046     21.44    148.71   Fri Mar 25 11:02:04 2005
  200    30792.02   82       153.9601     37.80    296.72   Fri Mar 25 11:02:42 2005
  300    32209.27   81       107.3642     54.21    451.34   Fri Mar 25 11:03:37 2005
  400    34962.83   78        87.4071     66.59    588.97   Fri Mar 25 11:04:44 2005
  500    31676.92   75        63.3538     91.87    742.71   Fri Mar 25 11:06:16 2005
  600    36032.69   73        60.0545     96.91    885.44   Fri Mar 25 11:07:54 2005
  700    35540.43   77        50.7720    114.63   1024.28   Fri Mar 25 11:09:49 2005
  800    33906.70   74        42.3834    137.32   1181.65   Fri Mar 25 11:12:06 2005
  900    34120.67   73        37.9119    153.51   1325.26   Fri Mar 25 11:14:41 2005
 1000    34802.37   74        34.8024    167.23   1465.26   Fri Mar 25 11:17:28 2005

with slab API changes and pageset patch:

Tasks    jobs/min  jti  jobs/min/task      real       cpu
    1      485.00  100       485.0000     12.00      1.96   Fri Mar 25 11:46:18 2005
  100    28000.96   89       280.0096     20.79    150.45   Fri Mar 25 11:46:39 2005
  200    32285.80   79       161.4290     36.05    293.37   Fri Mar 25 11:47:16 2005
  300    40424.15   84       134.7472     43.19    438.42   Fri Mar 25 11:47:59 2005
  400    39155.01   79        97.8875     59.46    590.05   Fri Mar 25 11:48:59 2005
  500    37881.25   82        75.7625     76.82    730.19   Fri Mar 25 11:50:16 2005
  600    39083.14   78        65.1386     89.35    872.79   Fri Mar 25 11:51:46 2005
  700    38627.83   77        55.1826    105.47   1022.46   Fri Mar 25 11:53:32 2005
  800    39631.94   78        49.5399    117.48   1169.94   Fri Mar 25 11:55:30 2005
  900    36903.70   79        41.0041    141.94   1310.78   Fri Mar 25 11:57:53 2005
 1000    36201.23   77        36.2012    160.77   1458.31   Fri Mar 25 12:00:34 2005

Signed-off-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Shobhit Dayal <shobhit@calsoftinc.com>
Signed-off-by: Shai Fultheim <Shai@Scalex86.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-21 18:46:16 -07:00

714 lines
16 KiB
C

/*
* linux/init/main.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*
* GK 2/5/95 - Changed to support mounting root fs via NFS
* Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96
* Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96
* Simplified starting of init: Michael A. Griffith <grif@acm.org>
*/
#define __KERNEL_SYSCALLS__
#include <linux/config.h>
#include <linux/types.h>
#include <linux/module.h>
#include <linux/proc_fs.h>
#include <linux/devfs_fs_kernel.h>
#include <linux/kernel.h>
#include <linux/syscalls.h>
#include <linux/string.h>
#include <linux/ctype.h>
#include <linux/delay.h>
#include <linux/utsname.h>
#include <linux/ioport.h>
#include <linux/init.h>
#include <linux/smp_lock.h>
#include <linux/initrd.h>
#include <linux/hdreg.h>
#include <linux/bootmem.h>
#include <linux/tty.h>
#include <linux/gfp.h>
#include <linux/percpu.h>
#include <linux/kmod.h>
#include <linux/kernel_stat.h>
#include <linux/security.h>
#include <linux/workqueue.h>
#include <linux/profile.h>
#include <linux/rcupdate.h>
#include <linux/moduleparam.h>
#include <linux/kallsyms.h>
#include <linux/writeback.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/efi.h>
#include <linux/unistd.h>
#include <linux/rmap.h>
#include <linux/mempolicy.h>
#include <linux/key.h>
#include <asm/io.h>
#include <asm/bugs.h>
#include <asm/setup.h>
/*
* This is one of the first .c files built. Error out early
* if we have compiler trouble..
*/
#if __GNUC__ == 2 && __GNUC_MINOR__ == 96
#ifdef CONFIG_FRAME_POINTER
#error This compiler cannot compile correctly with frame pointers enabled
#endif
#endif
#ifdef CONFIG_X86_LOCAL_APIC
#include <asm/smp.h>
#endif
/*
* Versions of gcc older than that listed below may actually compile
* and link okay, but the end product can have subtle run time bugs.
* To avoid associated bogus bug reports, we flatly refuse to compile
* with a gcc that is known to be too old from the very beginning.
*/
#if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 95)
#error Sorry, your GCC is too old. It builds incorrect kernels.
#endif
static int init(void *);
extern void init_IRQ(void);
extern void sock_init(void);
extern void fork_init(unsigned long);
extern void mca_init(void);
extern void sbus_init(void);
extern void sysctl_init(void);
extern void signals_init(void);
extern void buffer_init(void);
extern void pidhash_init(void);
extern void pidmap_init(void);
extern void prio_tree_init(void);
extern void radix_tree_init(void);
extern void free_initmem(void);
extern void populate_rootfs(void);
extern void driver_init(void);
extern void prepare_namespace(void);
#ifdef CONFIG_ACPI
extern void acpi_early_init(void);
#else
static inline void acpi_early_init(void) { }
#endif
#ifdef CONFIG_TC
extern void tc_init(void);
#endif
enum system_states system_state;
EXPORT_SYMBOL(system_state);
/*
* Boot command-line arguments
*/
#define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT
#define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT
extern void time_init(void);
/* Default late time init is NULL. archs can override this later. */
void (*late_time_init)(void);
extern void softirq_init(void);
/* Untouched command line (eg. for /proc) saved by arch-specific code. */
char saved_command_line[COMMAND_LINE_SIZE];
static char *execute_command;
/* Setup configured maximum number of CPUs to activate */
static unsigned int max_cpus = NR_CPUS;
/*
* Setup routine for controlling SMP activation
*
* Command-line option of "nosmp" or "maxcpus=0" will disable SMP
* activation entirely (the MPS table probe still happens, though).
*
* Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
* greater than 0, limits the maximum number of CPUs activated in
* SMP mode to <NUM>.
*/
static int __init nosmp(char *str)
{
max_cpus = 0;
return 1;
}
__setup("nosmp", nosmp);
static int __init maxcpus(char *str)
{
get_option(&str, &max_cpus);
return 1;
}
__setup("maxcpus=", maxcpus);
static char * argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
char * envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
static const char *panic_later, *panic_param;
extern struct obs_kernel_param __setup_start[], __setup_end[];
static int __init obsolete_checksetup(char *line)
{
struct obs_kernel_param *p;
p = __setup_start;
do {
int n = strlen(p->str);
if (!strncmp(line, p->str, n)) {
if (p->early) {
/* Already done in parse_early_param? (Needs
* exact match on param part) */
if (line[n] == '\0' || line[n] == '=')
return 1;
} else if (!p->setup_func) {
printk(KERN_WARNING "Parameter %s is obsolete,"
" ignored\n", p->str);
return 1;
} else if (p->setup_func(line + n))
return 1;
}
p++;
} while (p < __setup_end);
return 0;
}
/*
* This should be approx 2 Bo*oMips to start (note initial shift), and will
* still work even if initially too large, it will just take slightly longer
*/
unsigned long loops_per_jiffy = (1<<12);
EXPORT_SYMBOL(loops_per_jiffy);
static int __init debug_kernel(char *str)
{
if (*str)
return 0;
console_loglevel = 10;
return 1;
}
static int __init quiet_kernel(char *str)
{
if (*str)
return 0;
console_loglevel = 4;
return 1;
}
__setup("debug", debug_kernel);
__setup("quiet", quiet_kernel);
static int __init loglevel(char *str)
{
get_option(&str, &console_loglevel);
return 1;
}
__setup("loglevel=", loglevel);
/*
* Unknown boot options get handed to init, unless they look like
* failed parameters
*/
static int __init unknown_bootoption(char *param, char *val)
{
/* Change NUL term back to "=", to make "param" the whole string. */
if (val) {
/* param=val or param="val"? */
if (val == param+strlen(param)+1)
val[-1] = '=';
else if (val == param+strlen(param)+2) {
val[-2] = '=';
memmove(val-1, val, strlen(val)+1);
val--;
} else
BUG();
}
/* Handle obsolete-style parameters */
if (obsolete_checksetup(param))
return 0;
/*
* Preemptive maintenance for "why didn't my mispelled command
* line work?"
*/
if (strchr(param, '.') && (!val || strchr(param, '.') < val)) {
printk(KERN_ERR "Unknown boot option `%s': ignoring\n", param);
return 0;
}
if (panic_later)
return 0;
if (val) {
/* Environment option */
unsigned int i;
for (i = 0; envp_init[i]; i++) {
if (i == MAX_INIT_ENVS) {
panic_later = "Too many boot env vars at `%s'";
panic_param = param;
}
if (!strncmp(param, envp_init[i], val - param))
break;
}
envp_init[i] = param;
} else {
/* Command line option */
unsigned int i;
for (i = 0; argv_init[i]; i++) {
if (i == MAX_INIT_ARGS) {
panic_later = "Too many boot init vars at `%s'";
panic_param = param;
}
}
argv_init[i] = param;
}
return 0;
}
static int __init init_setup(char *str)
{
unsigned int i;
execute_command = str;
/*
* In case LILO is going to boot us with default command line,
* it prepends "auto" before the whole cmdline which makes
* the shell think it should execute a script with such name.
* So we ignore all arguments entered _before_ init=... [MJ]
*/
for (i = 1; i < MAX_INIT_ARGS; i++)
argv_init[i] = NULL;
return 1;
}
__setup("init=", init_setup);
extern void setup_arch(char **);
#ifndef CONFIG_SMP
#ifdef CONFIG_X86_LOCAL_APIC
static void __init smp_init(void)
{
APIC_init_uniprocessor();
}
#else
#define smp_init() do { } while (0)
#endif
static inline void setup_per_cpu_areas(void) { }
static inline void smp_prepare_cpus(unsigned int maxcpus) { }
#else
#ifdef __GENERIC_PER_CPU
unsigned long __per_cpu_offset[NR_CPUS];
EXPORT_SYMBOL(__per_cpu_offset);
static void __init setup_per_cpu_areas(void)
{
unsigned long size, i;
char *ptr;
/* Created by linker magic */
extern char __per_cpu_start[], __per_cpu_end[];
/* Copy section for each CPU (we discard the original) */
size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES);
#ifdef CONFIG_MODULES
if (size < PERCPU_ENOUGH_ROOM)
size = PERCPU_ENOUGH_ROOM;
#endif
ptr = alloc_bootmem(size * NR_CPUS);
for (i = 0; i < NR_CPUS; i++, ptr += size) {
__per_cpu_offset[i] = ptr - __per_cpu_start;
memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
}
}
#endif /* !__GENERIC_PER_CPU */
/* Called by boot processor to activate the rest. */
static void __init smp_init(void)
{
unsigned int i;
/* FIXME: This should be done in userspace --RR */
for_each_present_cpu(i) {
if (num_online_cpus() >= max_cpus)
break;
if (!cpu_online(i))
cpu_up(i);
}
/* Any cleanup work */
printk(KERN_INFO "Brought up %ld CPUs\n", (long)num_online_cpus());
smp_cpus_done(max_cpus);
#if 0
/* Get other processors into their bootup holding patterns. */
smp_commence();
#endif
}
#endif
/*
* We need to finalize in a non-__init function or else race conditions
* between the root thread and the init thread may cause start_kernel to
* be reaped by free_initmem before the root thread has proceeded to
* cpu_idle.
*
* gcc-3.4 accidentally inlines this function, so use noinline.
*/
static void noinline rest_init(void)
__releases(kernel_lock)
{
kernel_thread(init, NULL, CLONE_FS | CLONE_SIGHAND);
numa_default_policy();
unlock_kernel();
preempt_enable_no_resched();
cpu_idle();
}
/* Check for early params. */
static int __init do_early_param(char *param, char *val)
{
struct obs_kernel_param *p;
for (p = __setup_start; p < __setup_end; p++) {
if (p->early && strcmp(param, p->str) == 0) {
if (p->setup_func(val) != 0)
printk(KERN_WARNING
"Malformed early option '%s'\n", param);
}
}
/* We accept everything at this stage. */
return 0;
}
/* Arch code calls this early on, or if not, just before other parsing. */
void __init parse_early_param(void)
{
static __initdata int done = 0;
static __initdata char tmp_cmdline[COMMAND_LINE_SIZE];
if (done)
return;
/* All fall through to do_early_param. */
strlcpy(tmp_cmdline, saved_command_line, COMMAND_LINE_SIZE);
parse_args("early options", tmp_cmdline, NULL, 0, do_early_param);
done = 1;
}
/*
* Activate the first processor.
*/
asmlinkage void __init start_kernel(void)
{
char * command_line;
extern struct kernel_param __start___param[], __stop___param[];
/*
* Interrupts are still disabled. Do necessary setups, then
* enable them
*/
lock_kernel();
page_address_init();
printk(KERN_NOTICE);
printk(linux_banner);
setup_arch(&command_line);
setup_per_cpu_areas();
/*
* Mark the boot cpu "online" so that it can call console drivers in
* printk() and can access its per-cpu storage.
*/
smp_prepare_boot_cpu();
/*
* Set up the scheduler prior starting any interrupts (such as the
* timer interrupt). Full topology setup happens at smp_init()
* time - but meanwhile we still have a functioning scheduler.
*/
sched_init();
/*
* Disable preemption - early bootup scheduling is extremely
* fragile until we cpu_idle() for the first time.
*/
preempt_disable();
build_all_zonelists();
page_alloc_init();
printk(KERN_NOTICE "Kernel command line: %s\n", saved_command_line);
parse_early_param();
parse_args("Booting kernel", command_line, __start___param,
__stop___param - __start___param,
&unknown_bootoption);
sort_main_extable();
trap_init();
rcu_init();
init_IRQ();
pidhash_init();
init_timers();
softirq_init();
time_init();
/*
* HACK ALERT! This is early. We're enabling the console before
* we've done PCI setups etc, and console_init() must be aware of
* this. But we do want output early, in case something goes wrong.
*/
console_init();
if (panic_later)
panic(panic_later, panic_param);
profile_init();
local_irq_enable();
#ifdef CONFIG_BLK_DEV_INITRD
if (initrd_start && !initrd_below_start_ok &&
initrd_start < min_low_pfn << PAGE_SHIFT) {
printk(KERN_CRIT "initrd overwritten (0x%08lx < 0x%08lx) - "
"disabling it.\n",initrd_start,min_low_pfn << PAGE_SHIFT);
initrd_start = 0;
}
#endif
vfs_caches_init_early();
mem_init();
kmem_cache_init();
setup_per_cpu_pageset();
numa_policy_init();
if (late_time_init)
late_time_init();
calibrate_delay();
pidmap_init();
pgtable_cache_init();
prio_tree_init();
anon_vma_init();
#ifdef CONFIG_X86
if (efi_enabled)
efi_enter_virtual_mode();
#endif
fork_init(num_physpages);
proc_caches_init();
buffer_init();
unnamed_dev_init();
key_init();
security_init();
vfs_caches_init(num_physpages);
radix_tree_init();
signals_init();
/* rootfs populating might need page-writeback */
page_writeback_init();
#ifdef CONFIG_PROC_FS
proc_root_init();
#endif
cpuset_init();
check_bugs();
acpi_early_init(); /* before LAPIC and SMP init */
/* Do the rest non-__init'ed, we're now alive */
rest_init();
}
static int __initdata initcall_debug;
static int __init initcall_debug_setup(char *str)
{
initcall_debug = 1;
return 1;
}
__setup("initcall_debug", initcall_debug_setup);
struct task_struct *child_reaper = &init_task;
extern initcall_t __initcall_start[], __initcall_end[];
static void __init do_initcalls(void)
{
initcall_t *call;
int count = preempt_count();
for (call = __initcall_start; call < __initcall_end; call++) {
char *msg;
if (initcall_debug) {
printk(KERN_DEBUG "Calling initcall 0x%p", *call);
print_fn_descriptor_symbol(": %s()", (unsigned long) *call);
printk("\n");
}
(*call)();
msg = NULL;
if (preempt_count() != count) {
msg = "preemption imbalance";
preempt_count() = count;
}
if (irqs_disabled()) {
msg = "disabled interrupts";
local_irq_enable();
}
if (msg) {
printk(KERN_WARNING "error in initcall at 0x%p: "
"returned with %s\n", *call, msg);
}
}
/* Make sure there is no pending stuff from the initcall sequence */
flush_scheduled_work();
}
/*
* Ok, the machine is now initialized. None of the devices
* have been touched yet, but the CPU subsystem is up and
* running, and memory and process management works.
*
* Now we can finally start doing some real work..
*/
static void __init do_basic_setup(void)
{
/* drivers will send hotplug events */
init_workqueues();
usermodehelper_init();
driver_init();
#ifdef CONFIG_SYSCTL
sysctl_init();
#endif
/* Networking initialization needs a process context */
sock_init();
do_initcalls();
}
static void do_pre_smp_initcalls(void)
{
extern int spawn_ksoftirqd(void);
#ifdef CONFIG_SMP
extern int migration_init(void);
migration_init();
#endif
spawn_ksoftirqd();
}
static void run_init_process(char *init_filename)
{
argv_init[0] = init_filename;
execve(init_filename, argv_init, envp_init);
}
static inline void fixup_cpu_present_map(void)
{
#ifdef CONFIG_SMP
int i;
/*
* If arch is not hotplug ready and did not populate
* cpu_present_map, just make cpu_present_map same as cpu_possible_map
* for other cpu bringup code to function as normal. e.g smp_init() etc.
*/
if (cpus_empty(cpu_present_map)) {
for_each_cpu(i) {
cpu_set(i, cpu_present_map);
}
}
#endif
}
static int init(void * unused)
{
lock_kernel();
/*
* init can run on any cpu.
*/
set_cpus_allowed(current, CPU_MASK_ALL);
/*
* Tell the world that we're going to be the grim
* reaper of innocent orphaned children.
*
* We don't want people to have to make incorrect
* assumptions about where in the task array this
* can be found.
*/
child_reaper = current;
/* Sets up cpus_possible() */
smp_prepare_cpus(max_cpus);
do_pre_smp_initcalls();
fixup_cpu_present_map();
smp_init();
sched_init_smp();
cpuset_init_smp();
/*
* Do this before initcalls, because some drivers want to access
* firmware files.
*/
populate_rootfs();
do_basic_setup();
/*
* check if there is an early userspace init. If yes, let it do all
* the work
*/
if (sys_access((const char __user *) "/init", 0) == 0)
execute_command = "/init";
else
prepare_namespace();
/*
* Ok, we have completed the initial bootup, and
* we're essentially up and running. Get rid of the
* initmem segments and start the user-mode stuff..
*/
free_initmem();
unlock_kernel();
system_state = SYSTEM_RUNNING;
numa_default_policy();
if (sys_open((const char __user *) "/dev/console", O_RDWR, 0) < 0)
printk(KERN_WARNING "Warning: unable to open an initial console.\n");
(void) sys_dup(0);
(void) sys_dup(0);
/*
* We try each of these until one succeeds.
*
* The Bourne shell can be used instead of init if we are
* trying to recover a really broken machine.
*/
if (execute_command)
run_init_process(execute_command);
run_init_process("/sbin/init");
run_init_process("/etc/init");
run_init_process("/bin/init");
run_init_process("/bin/sh");
panic("No init found. Try passing init= option to kernel.");
}