linux SMP启动代码分析

If one replicates an entire CPU to execute a second thread, then the technique is known as   multi-processing .

If one replicates only a portion of a CPU to execute a second thread, then the technique is known as   multi-threading . (关于MP详细的可以查看An Overview of MIPS Multi-Threading )

红色部分是共用的，蓝色部分是私有的，灰色部分跟实现有关

start_kernel（）

-->boot_cpu_init()

        --->int cpu = smp_processor_id();

                            ---->raw_smp_processor_id()

                                    --->#define raw_smp_processor_id() (current_thread_info()->cpu)

        ---->set_cpu_online(cpu, true);

---->setup_arch()

        --->prom_init()

                --->board_ebase_setup = &bmips_ebase_setup;

                --->register_smp_ops(& bmips_smp_ops );

        ---->plat_smp_setup()

                ----> mp_ops->smp_setup();     =    bmips_smp_setup()

#define NR_CPUS        CONFIG_NR_CPUS

struct plat_smp_ops   bmips_smp_ops   = {

    .smp_setup        =   bmips_smp_setup ,

    .prepare_cpus        = bmips_prepare_cpus,

    .boot_secondary        = bmips_boot_secondary,

    .smp_finish        = bmips_smp_finish,

    .init_secondary        = bmips_init_secondary,

    .cpus_done        = bmips_cpus_done,

    .send_ipi_single    = bmips_send_ipi_single,

    .send_ipi_mask        = bmips_send_ipi_mask,

#ifdef CONFIG_HOTPLUG_CPU

    .cpu_disable        = bmips_cpu_disable,

    .cpu_die        = bmips_cpu_die,

#endif

};

static void __init   bmips_smp_setup (void)

{

     /* arbitration priority */

    clear_c0_brcm_cmt_ctrl(0x30);

    /* NBK and weak order flags */

    set_c0_brcm_config_0(0x30000);

    /*

     * MIPS interrupts 0,1 (SW INT 0,1) cross over to the other thread

     * MIPS interrupt 2 (HW INT 0) is the CPU0 L1 controller output

     * MIPS interrupt 3 (HW INT 1) is the CPU1 L1 controller output

     */

    change_c0_brcm_cmt_intr(0xf8018000,

        (0x02 << 27) | (0x03 << 15));

    /* single core, 2 threads (2 pipelines) */

    max_cpus = 2;

        if (!bmips_smp_enabled)

        max_cpus = 1;

    /* this can be overridden by the BSP */

    if (!board_ebase_setup)

        board_ebase_setup = &bmips_ebase_setup;

    for (i = 0; i < max_cpus; i++) {

        __cpu_number_map[i] = 1;

        __cpu_logical_map[i] = 1;

        set_cpu_possible(i, 1);

        set_cpu_present(i, 1);

    }

}

/*

* SMTC Linux requires shutting-down microthread scheduling

* during CP0 register read-modify-write sequences.

*/

#define   __BUILD_SET_C0 (name)                    \

static inline unsigned int                    \

set_c0_##name(unsigned int set)                    \

{                                \

    unsigned int res, new;                    \

                                \

    res = read_c0_##name();                    \

    new = res | set;                    \

    write_c0_##name(new);                    \

                                \

    return res;                        \

}                                \

                                \

static inline unsigned int                    \

clear_c0_##name(unsigned int clear)                \

{                                \

    unsigned int res, new;                    \

                                \

    res = read_c0_##name();                    \

    new = res & ~clear;                    \

    write_c0_##name(new);                    \

                                \

    return res;                        \

}                                \

                                \

static inline unsigned int                    \

change_c0_##name(unsigned int change, unsigned int val)        \

{                                \

    unsigned int res, new;                    \

                                \

    res = read_c0_##name();                    \

    new = res & ~change;                    \

    new |= (val & change);                    \

    write_c0_##name(new);                    \

                                \

    return res;                        \

}

/* BMIPS43xx */

#define read_c0_ brcm_cmt_intr ()        __read_32bit_c0_register($22, 1)

#define write_c0_brcm_cmt_intr(val)    __write_32bit_c0_register($22, 1, val)

#define read_c0_ brcm_cmt_ctrl ()        __read_32bit_c0_register($22, 2)

#define write_c0_brcm_cmt_ctrl(val)    __write_32bit_c0_register($22, 2, val)

#define read_c0_ brcm_cmt_local ()    __read_32bit_c0_register($22, 3)

#define write_c0_brcm_cmt_local(val)    __write_32bit_c0_register($22, 3, val)

__BUILD_SET_C0( brcm_cmt_intr )

__BUILD_SET_C0( brcm_cmt_ctrl )

start_kernel（）

------>rest_init()

            --->kernel_init()

                    ---->smp_init()

                             ---->cpu_up()

                                         ----->_cpu_up()

                                                     --->__cpu_up()

int __cpuinit __cpu_up(unsigned int cpu)

{

    struct task_struct *idle;

    /*

     * Processor goes to start_secondary(), sets online flag

     * The following code is purely to make sure

     * Linux can schedule processes on this slave.

     */

    if (!cpu_idle_thread[cpu]) {

        /*

         * Schedule work item to avoid forking user task

         * Ported from arch/x86/kernel/smpboot.c

         */

        struct create_idle c_idle = {

            .cpu    = cpu,

            .done   = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),

        };

        INIT_WORK_ONSTACK(&c_idle.work,  do_fork_idle );

        schedule_work(&c_idle.work);

        wait_for_completion(&c_idle.done);

        idle = cpu_idle_thread[cpu] = c_idle.idle;

        if (IS_ERR(idle))

            panic(KERN_ERR "Fork failed for CPU %d", cpu);

    } else {

        idle = cpu_idle_thread[cpu];

        init_idle(idle, cpu);

    }

    mp_ops-> boot_secondary (cpu, idle);

    /*

     * Trust is futile.  We should really have timeouts ...

     */

    while (!cpu_isset(cpu, cpu_callin_map))

        udelay(100);

    cpu_set(cpu, cpu_online_map);

    return 0;

}

static void   bmips_boot_secondary (int cpu, struct task_struct *idle)

{

    bmips_smp_boot_sp = __KSTK_TOS(idle);

    bmips_smp_boot_gp = (unsigned long)task_thread_info(idle);

    mb();

     /*

     * Initial boot sequence for secondary CPU:

     *   bmips_reset_nmi_vec @ a000_0000 ->

     *   bmips_smp_entry ->

     *   plat_wired_tlb_setup (cached function call; optional) ->

     *   start_secondary (cached jump)

     *

     * Warm restart sequence:

     *   play_dead WAIT loop ->

     *   bmips_smp_int_vec @ BMIPS_WARM_RESTART_VEC ->

     *   eret to play_dead ->

     *   bmips_secondary_reentry ->

     *   start_secondary

     */

    pr_info("SMP: Booting CPU%d...\n", cpu);

    if (cpumask_test_cpu(cpu, &bmips_booted_mask)) {

        /* kseg1 might not exist if this CPU enabled XKS01 */

        bmips_set_reset_vec(cpu, RESET_FROM_KSEG0);

        bmips_send_ipi_single(cpu, 0);  //如果cpu已经在运行了，则软复位cpu

    } else {

        bmips_set_reset_vec(cpu, RESET_FROM_KSEG1);   //设置cpu复位的异常向量位置

#if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380)

        set_c0_brcm_cmt_ctrl(0x01);     //复位cpu

#elif defined(CONFIG_BCM7435A0)

        if (cpu & 0x01)

            write_c0_brcm_action(ACTION_BOOT_THREAD(cpu));

        else {

            /*

             * core N thread 0 was already booted; just

             * pulse the NMI line

             */

            bmips_write_zscm_reg(0x210, 0xc0000000);

            udelay(10);

            bmips_write_zscm_reg(0x210, 0x00);

        }

#elif defined(CONFIG_CPU_BMIPS5000)

        write_c0_brcm_action(ACTION_BOOT_THREAD(cpu));

#endif

        cpumask_set_cpu(cpu, &bmips_booted_mask);

    }

}

static void   bmips_set_reset_vec (int cpu, u32 val)

{

    struct reset_vec_info info;

    if (current_cpu_type() == CPU_BMIPS5000) {

        /* this needs to run from CPU0 (which is always online) */

        info.cpu = cpu;

        info.val = val;

        bmips_set_reset_vec_remote(&info);

    } else {

        void __iomem *cbr = BMIPS_GET_CBR();

        if (cpu == 0)

            __raw_writel(val, cbr + BMIPS_RELO_VECTOR_CONTROL_0);

        else {

            if (current_cpu_type() != CPU_BMIPS4380)

                return;

            __raw_writel(val, cbr +   BMIPS_RELO_VECTOR_CONTROL_1 );

        }

    }

    __sync();

    back_to_back_c0_hazard();

}

复位异常向量是如何安装的呢

static inline void __cpuinit   bmips_nmi_handler_setup (void)

{

    bmips_wr_vec(BMIPS_NMI_RESET_VEC, & bmips_reset_nmi_vec ,

        &bmips_reset_nmi_vec_end);

    bmips_wr_vec(BMIPS_WARM_RESTART_VEC, &bmips_smp_int_vec,

        &bmips_smp_int_vec_end);

}

void __cpuinit bmips_ebase_setup(void)

{

    ....

    board_nmi_handler_setup = &bmips_nmi_handler_setup;

}

trap_init()

{

     ....

        if (board_nmi_handler_setup)

        board_nmi_handler_setup();

}

而trap_init（）是在start_kernel（）中调用的。

cpu复位后进入异常处理执行bmips_reset_nmi_vec，下面来看看bmips_reset_nmi_vec的实现：

arch/mips/kernel/bmips_vec.S

/***********************************************************************

* Reset/NMI vector

* For BMIPS processors that can relocate their exception vectors, this

* entire function gets copied to 0x8000_0000.

***********************************************************************/

NESTED( bmips_reset_nmi_vec , PT_SIZE, sp)

    .set    push

    .set    noat

    .align    4

#ifdef CONFIG_SMP

    /* if the NMI bit is clear, assume this is a soft reset */

    li    k1, (1 << 19)

    mfc0    k0, CP0_STATUS

    and    k0, k1

    beqz    k0, soft_reset

#if defined(CONFIG_CPU_BMIPS5000)

    /* if we're not on core 0, this must be the SMP boot signal */

    li    k1, (3 << 25)

    mfc0    k0, $22

    and    k0, k1

    bnez    k0, bmips_smp_entry

#endif

#endif /* CONFIG_SMP */

    /* nope, it's just a regular NMI */

    SAVE_ALL

    move    a0, sp

    /* clear EXL, ERL, BEV so that TLB refills still work */

    mfc0    k0, CP0_STATUS

    li    k1, ST0_ERL | ST0_EXL | ST0_BEV | ST0_IE

    or    k0, k1

    xor    k0, k1

    mtc0    k0, CP0_STATUS

    BARRIER

    /* jump to the NMI handler function */

    la    k0, nmi_handler

    jr    k0

    RESTORE_ALL

    .set    mips3

    eret

/***********************************************************************

* CPU1 reset vector (used for the initial and warm boot only)

* This is still part of bmips_reset_nmi_vec().

***********************************************************************/

#ifdef CONFIG_SMP

soft_reset:

#if defined(CONFIG_CPU_BMIPS5000) && defined(CONFIG_BCM7435)

    /* if running on TP 1, jump  to  bmips_smp_entry */

    mfc0    k0, $22

    li    k1, (1 << 24)

    and    k1, k0

    bnez    k1, bmips_smp_entry

    nop

    /*

     * running on TP0, can not be core 0 (the boot core).

     * Check for soft reset.  Indicates a warm boot

     */

    mfc0    k0, $12

    li    k1, (1 << 20)

    and    k0, k1

    beqz    k0, bmips_smp_entry

    /*

     * Warm boot.

     * Cache init is only done on TP0

     */

    la    k0, bmips_5xxx_init

    jalr    k0

    nop

#if !defined(CONFIG_BCM7435A0)

    b    bmips_smp_entry

    nop

#else

    /* wait for nmi interrupt from start_secondary */

1:

    wait

    b    1b

    nop

#endif

#endif

bmips_smp_entry:

    /* set up CP0 STATUS; enable FPU */

    li    k0, 0x30000000

    mtc0    k0, CP0_STATUS

    BARRIER

    /* set local CP0 CONFIG to make kseg0 cacheable, write-back */

    mfc0    k0, CP0_CONFIG

    ori    k0, 0x07

    xori    k0, 0x04

    mtc0    k0, CP0_CONFIG

#if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380)

    /* initialize CPU1's local I-cache */

    li    k0, 0x80000000

    li    k1, 0x80010000

    mtc0    zero, $28

    mtc0    zero, $28, 1

    BARRIER

1:    cache    Index_Store_Tag_I, 0(k0)

    addiu    k0, 16

    bne    k0, k1, 1b

#elif defined(CONFIG_CPU_BMIPS5000)

    /* set exception vector base */

    la    k0, ebase

    lw    k0, 0(k0)

    mtc0    k0, $15, 1

    BARRIER

#endif

    /* jump back to kseg0 in case we need to remap the kseg1 area */

    la    k0, 1f

    jr    k0

1:

    la    k0, bmips_enable_xks01

    jalr    k0

    /* use temporary stack to set up upper memory TLB */

    li    sp, BMIPS_WARM_RESTART_VEC

    la    k0, plat_wired_tlb_setup

    jalr    k0

    /* switch to permanent stack and continue booting */

    .global    bmips_secondary_reentry

bmips_secondary_reentry:

    la    k0, bmips_smp_boot_sp

    lw    sp, 0(k0)

    la    k0, bmips_smp_boot_gp

    lw    gp, 0(k0)

    la    k0, start_secondary

    jr    k0

#endif /* CONFIG_SMP */

    .align    4

    .global    bmips_reset_nmi_vec_end

bmips_reset_nmi_vec_end:

END(bmips_reset_nmi_vec)

    .set    pop

    .previous

arch/mips/kernel/smp.c

/*

* First C code run on the secondary CPUs after being started up by

* the master.

*/

asmlinkage __cpuinit void   start_secondary (void)

{

    unsigned int cpu;

#ifdef CONFIG_MIPS_MT_SMTC

    /* Only do cpu_probe for first TC of CPU */

    if ((read_c0_tcbind() & TCBIND_CURTC) == 0)

#endif /* CONFIG_MIPS_MT_SMTC */

    cpu_probe();

    cpu_report();

    per_cpu_trap_init();

    mips_clockevent_init();

    mp_ops->init_secondary();

    /*

     * XXX parity protection should be folded in here when it's converted

     * to an option instead of something based on .cputype

     */

    calibrate_delay();

    preempt_disable();

    cpu = smp_processor_id();

    cpu_data[cpu].udelay_val = loops_per_jiffy;

    notify_cpu_starting(cpu);

    mp_ops->smp_finish();

    set_cpu_sibling_map(cpu);

    cpu_set(cpu, cpu_callin_map);

    synchronise_count_slave();

     cpu_idle ();

}