飞腾FT2000+的NUMA拓扑问题

飞腾FT2000+的NUMA拓扑问题

现象

[root@localhost]#lscpu
Architecture: aarch64
Byte Order: Little Endian
CPU(s): 64
On-line CPU(s) list: 0-63
Thread(s) per core: 1
Core(s) per socket: 4 //?
Socket(s): 16 //？
NUMA node(s): 1 //？
Vendor ID: 0x70
Model: 2
Stepping: 0x1
BogoMIPS: 100.00
NUMA node0 CPU(s): 0-63 //?
Flags: fp asimd evtstrm crc32 cpuid
上述lscpu命令获取服务器的芯片结构和NUMA结构特性发生错误。

FT2000+/64芯片

这是一款单处理器的服务器芯片，全芯片64个硬核core，没有实现超线（即每个core只有一个thread）。
全芯片有八个NUMA结点0～7，每个NUMA结点有8个cores和一个MCU。正确的描述如下：
[root@localhost]# lscpu
Architecture: aarch64
Byte Order: Little Endian
CPU(s): 64
On-line CPU(s) list: 0-63
Thread(s) per core: 1
Core(s) per socket: 64 // ok
Socket(s): 1 //ok
NUMA node(s): 8 //ok
Vendor ID: 0x70
Model: 2
Stepping: 0x1
BogoMIPS: 100.00
NUMA node0 CPU(s): 0-7 //ok
NUMA node1 CPU(s): 8-15 //ok
NUMA node2 CPU(s): 16-23
NUMA node3 CPU(s): 24-31
NUMA node4 CPU(s): 32-39
NUMA node5 CPU(s): 40-47
NUMA node6 CPU(s): 48-55
NUMA node7 CPU(s): 56-63
Flags: fp asimd evtstrm crc32 cpuid

内核补丁（可以下载）

CPU和NUMA结构描述本应该在UEFI的ACPI表或者uboot的设备树来描述。当机器固件还不成熟时，可以采用下面补丁：

// arch/arm64/include/asm/cputype.h
// 增加对FT2000+芯片的识别函数
#define ARM_CPU_IMP_PHYTIUM             0x70
#define ARM_CPU_PART_PHY_S662           0x662
#define MIDR_PHY_S_FT2000PLUS			MIDR_CPU_MODEL(ARM_CPU_IMP_PHYTIUM,\
										 ARM_CPU_PART_PHY_S662)
static inline bool __attribute_const__ is_phytium_s_ft2000plus(void)
{
	if ((read_cpuid_id() & MIDR_CPU_MODEL_MASK) != 
								MIDR_PHY_S_FT2000PLUS)
               return false;
       return true;
}


//arch/arm64/kernel/smp.c
//往NUMA结构中分配相应的CPU
static void __init phytium_s_ft2000plus_parse_and_init_cpus(void)
{
       int i;
       for (i = 0; i < nr_cpu_ids; i++)
               early_map_cpu_to_node(i, i>>3);
       cpu_count = nr_cpu_ids>64?64:nr_cpu_ids;
       bootcpu_valid = true;      
}
/*在设备树或acpi失败之后*/
       if(is_phytium_s_ft2000plus())
             phytium_s_ft2000plus_parse_and_init_cpus();

//arch/arm64/kernel/topology.c
//增加飞腾的mpidr芯片解析
if(is_phytium_s_ft2000plus()) {
	cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0) | 
						  MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 
						  MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16;
	cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 3);
}

//arch/arm64/mm/numa.c
//对numa结构初始化
static int __init phytium_s_ft2000plus_numa_init(void)
{
        int ret = -1;
        u32 nid, from, to;
        u32 distance;
        struct memblock_region *mblk;
        for(nid=0; nid<8; nid++)
                node_set(nid, numa_nodes_parsed);
        for_each_memblock(memory, mblk) {
                nid = mblk->base >> 40;
                if(nid > 7) {
                	pr_warn("unknown memblk rang [%#018Lx, %#018Lx]\n", 
                			mblk->base, mblk->base + mblk->size);
                    continue;
                }
                if(0 > numa_add_memblk(nid, 
                			mblk->base, mblk->base + mblk->size))
                   	continue;
                ret = 0;
    	}
    	
        if(ret < 0)
               return ret;
               
        for (from=0; from<8; from++)
                for (to=0; to<8; to++) {                      
                        distance = REMOTE_DISTANCE;//20`
                        if ( (from == to)                      ||
                            ((from==0) && (to==4))             ||
                            ((from==4) && (to==0))             ||
                            ((from==1) && (to==5))             ||
                            ((from==5) && (to==1))             ||
                            ((from==3) && (to==7))             ||
                            ((from==7) && (to==3))             ||
                            ((from==2) && (to==6))             ||
                            ((from==6) && (to==2)) )
                               distance = LOCAL_DISTANCE;//10
                        numa_set_distance(from, to, distance);
                }
        return 0;
}


/*在arm64_numa_init函数添加入口*/
	if (is_phytium_s_ft2000plus() && 
				!numa_init(phytium_s_ft2000plus_numa_init))
		return;