源码编译安装 hwloc 查看cpu pcie 以及 cache 拓扑

Eloudy

已于 2024-11-20 15:38:27 修改

阅读量451

点赞数 1

文章标签： GPU cuda numa HPC MPI

于 2024-11-20 14:25:20 首次发布

本文链接：https://blog.csdn.net/eloudy/article/details/143909220

版权

1，下载 hwloc 源码

$ git clone https://github.com/open-mpi/hwloc.git

cd hwloc/
$ git checkout hwloc-2.11.2

2，运行 auto-tool

$ ./autogen.sh
mkdir build
cd build

3，配置项目

简单配置

$ ../configure --prefix=/home/hipper/ex_hwloc/tmp2/localh

4，编译安装

$ make -j
$ make install

5，运行现成工具

bin$ ./hwloc-ls

6，基于hwloc 的源码示例

Makefile

hello: hello_hwloc.c
	gcc $< -o $@ $(INC) $(LD_FLAGS)

INC := -I../../localh/include/ 
LD_FLAGS := -L../../localh/lib/ -lhwloc

.PHONY: clean
clean:
	-rm -rf hello

hello_hwloc.c

/* Example hwloc API program.
 *
 * See other examples under doc/examples/ in the source tree
 * for more details.
 *
 * Copyright (c) 2009-2016 Inria. All rights reserved.
 * Copyright (c) 2009-2011 Universit?eacute; Bordeaux
 * Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
 * See COPYING in top-level directory.
 *
 * hwloc-hello.c
 */
#include "hwloc.h"
#include <errno.h>
#include <stdio.h>
#include <string.h>

static void print_children(hwloc_topology_t topology, hwloc_obj_t obj,
                           int depth) {
  char type[32], attr[1024];
  unsigned i;
  hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
  printf("%*s%s", 2 * depth, "", type);
  if (obj->os_index != (unsigned)-1)
    printf("#%u", obj->os_index);
  hwloc_obj_attr_snprintf(attr, sizeof(attr), obj, " ", 0);
  if (*attr)
    printf("(%s)", attr);
  printf("\n");
  for (i = 0; i < obj->arity; i++) {
    print_children(topology, obj->children[i], depth + 1);
  }
}

int main(void) {
  int depth;
  unsigned i, n;
  unsigned long size;
  int levels;
  char string[128];
  int topodepth;
  void *m;
  hwloc_topology_t topology;
  hwloc_cpuset_t cpuset;
  hwloc_obj_t obj;
  /* Allocate and initialize topology object. */
  hwloc_topology_init(&topology);
  /* ... Optionally, put detection configuration here to ignore
  some objects types, define a synthetic topology, etc....
  The default is to detect all the objects of the machine that
  the caller is allowed to access. See Configure Topology
  Detection. */
  /* Perform the topology detection. */
  hwloc_topology_load(topology);
  /* Optionally, get some additional topology information
  in case we need the topology depth later. */
  topodepth = hwloc_topology_get_depth(topology);
  /*****************************************************************
   * First example:
   * Walk the topology with an array style, from level 0 (always
   * the system level) to the lowest level (always the proc level).
   *****************************************************************/
  for (depth = 0; depth < topodepth; depth++) {
    printf("*** Objects at level %d\n", depth);
    for (i = 0; i < hwloc_get_nbobjs_by_depth(topology, depth); i++) {
      hwloc_obj_type_snprintf(string, sizeof(string),
                              hwloc_get_obj_by_depth(topology, depth, i), 0);
      printf("Index %u: %s\n", i, string);
    }
  }
  /*****************************************************************
   * Second example:
   * Walk the topology with a tree style.
   *****************************************************************/
  printf("*** Printing overall tree\n");
  print_children(topology, hwloc_get_root_obj(topology), 0);
  /*****************************************************************
   * Third example:
   * Print the number of packages.
   *****************************************************************/
  depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PACKAGE);
  if (depth == HWLOC_TYPE_DEPTH_UNKNOWN) {
    printf("*** The number of packages is unknown\n");
  } else {
    printf("*** %u package(s)\n", hwloc_get_nbobjs_by_depth(topology, depth));
  }
  /*****************************************************************
   * Fourth example:
   * Compute the amount of cache that the first logical processor
   * has above it.
   *****************************************************************/
  levels = 0;
  size = 0;
  for (obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 0); obj;
       obj = obj->parent)
    if (hwloc_obj_type_is_cache(obj->type)) {
      levels++;
      size += obj->attr->cache.size;
    }
  printf("*** Logical processor 0 has %d caches totaling %luKB\n", levels,
         size / 1024);
  /*****************************************************************
   * Fifth example:
   * Bind to only one thread of the last core of the machine.
   *
   * First find out where cores are, or else smaller sets of CPUs if
   * the OS doesn't have the notion of a "core".
   *****************************************************************/
  depth = hwloc_get_type_or_below_depth(topology, HWLOC_OBJ_CORE);
  /* Get last core. */
  obj = hwloc_get_obj_by_depth(topology, depth,
                               hwloc_get_nbobjs_by_depth(topology, depth) - 1);
  if (obj) {
    /* Get a copy of its cpuset that we may modify. */
    cpuset = hwloc_bitmap_dup(obj->cpuset);
    /* Get only one logical processor (in case the core is
    SMT/hyper-threaded). */
    hwloc_bitmap_singlify(cpuset);
    /* And try to bind ourself there. */
    if (hwloc_set_cpubind(topology, cpuset, 0)) {
      char *str;
      int error = errno;
      hwloc_bitmap_asprintf(&str, obj->cpuset);
      printf("Couldn't bind to cpuset %s: %s\n", str, strerror(error));
      free(str);
    }
    /* Free our cpuset copy */
    hwloc_bitmap_free(cpuset);
  }
  /*****************************************************************
   * Sixth example:
   * Allocate some memory on the last NUMA node, bind some existing
   * memory to the last NUMA node.
   *****************************************************************/
  /* Get last node. There's always at least one. */
  n = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NUMANODE);
  obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, n - 1);
  size = 1024 * 1024;
  m = hwloc_alloc_membind(topology, size, obj->nodeset, HWLOC_MEMBIND_BIND,
                          HWLOC_MEMBIND_BYNODESET);
  hwloc_free(topology, m, size);
  m = malloc(size);
  hwloc_set_area_membind(topology, m, size, obj->nodeset, HWLOC_MEMBIND_BIND,
                         HWLOC_MEMBIND_BYNODESET);
  free(m);
  /* Destroy topology object. */
  hwloc_topology_destroy(topology);
  return 0;
}

编译运行：

make

./hello

输出：

7，一台2路cpu8GPU带nvlink的机器 hwloc-ls

localhw/bin$ ./hwloc-ls

localhw/bin$ ./hwloc-ls
Machine (2016GiB total)
  Package L#0
    Group0 L#0
      NUMANode L#0 (P#0 252GiB)
      L3 L#0 (16MiB)
        L2 L#0 (512KiB) + L1d L#0 (32KiB) + L1i L#0 (32KiB) + Core L#0
          PU L#0 (P#0)
          PU L#1 (P#128)
        L2 L#1 (512KiB) + L1d L#1 (32KiB) + L1i L#1 (32KiB) + Core L#1
          PU L#2 (P#1)
          PU L#3 (P#129)
        L2 L#2 (512KiB) + L1d L#2 (32KiB) + L1i L#2 (32KiB) + Core L#2
          PU L#4 (P#2)
          PU L#5 (P#130)
        L2 L#3 (512KiB) + L1d L#3 (32KiB) + L1i L#3 (32KiB) + Core L#3
          PU L#6 (P#3)
          PU L#7 (P#131)
      L3 L#1 (16MiB)
        L2 L#4 (512KiB) + L1d L#4 (32KiB) + L1i L#4 (32KiB) + Core L#4
          PU L#8 (P#4)
          PU L#9 (P#132)
        L2 L#5 (512KiB) + L1d L#5 (32KiB) + L1i L#5 (32KiB) + Core L#5
          PU L#10 (P#5)
          PU L#11 (P#133)
        L2 L#6 (512KiB) + L1d L#6 (32KiB) + L1i L#6 (32KiB) + Core L#6
          PU L#12 (P#6)
          PU L#13 (P#134)
        L2 L#7 (512KiB) + L1d L#7 (32KiB) + L1i L#7 (32KiB) + Core L#7
          PU L#14 (P#7)
          PU L#15 (P#135)
      L3 L#2 (16MiB)
        L2 L#8 (512KiB) + L1d L#8 (32KiB) + L1i L#8 (32KiB) + Core L#8
          PU L#16 (P#8)
          PU L#17 (P#136)
        L2 L#9 (512KiB) + L1d L#9 (32KiB) + L1i L#9 (32KiB) + Core L#9
          PU L#18 (P#9)
          PU L#19 (P#137)
        L2 L#10 (512KiB) + L1d L#10 (32KiB) + L1i L#10 (32KiB) + Core L#10
          PU L#20 (P#10)
          PU L#21 (P#138)
        L2 L#11 (512KiB) + L1d L#11 (32KiB) + L1i L#11 (32KiB) + Core L#11
          PU L#22 (P#11)
          PU L#23 (P#139)
      L3 L#3 (16MiB)
        L2 L#12 (512KiB) + L1d L#12 (32KiB) + L1i L#12 (32KiB) + Core L#12
          PU L#24 (P#12)
          PU L#25 (P#140)
        L2 L#13 (512KiB) + L1d L#13 (32KiB) + L1i L#13 (32KiB) + Core L#13
          PU L#26 (P#13)
          PU L#27 (P#141)
        L2 L#14 (512KiB) + L1d L#14 (32KiB) + L1i L#14 (32KiB) + Core L#14
          PU L#28 (P#14)
          PU L#29 (P#142)
        L2 L#15 (512KiB) + L1d L#15 (32KiB) + L1i L#15 (32KiB) + Core L#15
          PU L#30 (P#15)
          PU L#31 (P#143)
      HostBridge
        PCIBridge
          PCI 61:00.0 (Ethernet)
            Net "enp97s0f0"
            OFED "mlx5_4"
          PCI 61:00.1 (Ethernet)
            Net "enp97s0f1"
            OFED "mlx5_5"
        PCIBridge
          PCIBridge
            PCI 63:00.0 (VGA)
    Group0 L#1
      NUMANode L#1 (P#1 252GiB)
      L3 L#4 (16MiB)
        L2 L#16 (512KiB) + L1d L#16 (32KiB) + L1i L#16 (32KiB) + Core L#16
          PU L#32 (P#16)
          PU L#33 (P#144)
        L2 L#17 (512KiB) + L1d L#17 (32KiB) + L1i L#17 (32KiB) + Core L#17
          PU L#34 (P#17)
          PU L#35 (P#145)
        L2 L#18 (512KiB) + L1d L#18 (32KiB) + L1i L#18 (32KiB) + Core L#18
          PU L#36 (P#18)
          PU L#37 (P#146)
        L2 L#19 (512KiB) + L1d L#19 (32KiB) + L1i L#19 (32KiB) + Core L#19
          PU L#38 (P#19)
          PU L#39 (P#147)
      L3 L#5 (16MiB)
        L2 L#20 (512KiB) + L1d L#20 (32KiB) + L1i L#20 (32KiB) + Core L#20
          PU L#40 (P#20)
          PU L#41 (P#148)
        L2 L#21 (512KiB) + L1d L#21 (32KiB) + L1i L#21 (32KiB) + Core L#21
          PU L#42 (P#21)
          PU L#43 (P#149)
        L2 L#22 (512KiB) + L1d L#22 (32KiB) + L1i L#22 (32KiB) + Core L#22
          PU L#44 (P#22)
          PU L#45 (P#150)
        L2 L#23 (512KiB) + L1d L#23 (32KiB) + L1i L#23 (32KiB) + Core L#23
          PU L#46 (P#23)
          PU L#47 (P#151)
      L3 L#6 (16MiB)
        L2 L#24 (512KiB) + L1d L#24 (32KiB) + L1i L#24 (32KiB) + Core L#24
          PU L#48 (P#24)
          PU L#49 (P#152)
        L2 L#25 (512KiB) + L1d L#25 (32KiB) + L1i L#25 (32KiB) + Core L#25
          PU L#50 (P#25)
          PU L#51 (P#153)
        L2 L#26 (512KiB) + L1d L#26 (32KiB) + L1i L#26 (32KiB) + Core L#26
          PU L#52 (P#26)
          PU L#53 (P#154)
        L2 L#27 (512KiB) + L1d L#27 (32KiB) + L1i L#27 (32KiB) + Core L#27
          PU L#54 (P#27)
          PU L#55 (P#155)
      L3 L#7 (16MiB)
        L2 L#28 (512KiB) + L1d L#28 (32KiB) + L1i L#28 (32KiB) + Core L#28
          PU L#56 (P#28)
          PU L#57 (P#156)
        L2 L#29 (512KiB) + L1d L#29 (32KiB) + L1i L#29 (32KiB) + Core L#29
          PU L#58 (P#29)
          PU L#59 (P#157)
        L2 L#30 (512KiB) + L1d L#30 (32KiB) + L1i L#30 (32KiB) + Core L#30
          PU L#60 (P#30)
          PU L#61 (P#158)
        L2 L#31 (512KiB) + L1d L#31 (32KiB) + L1i L#31 (32KiB) + Core L#31
          PU L#62 (P#31)
          PU L#63 (P#159)
      HostBridge
        PCIBridge
          PCIBridge
            PCIBridge
              PCIBridge
                PCIBridge
                  PCIBridge
                    PCIBridge
                      PCI 47:00.0 (3D)
                        CoProc(CUDA) "cuda2"
                        CoProc(OpenCL) "opencl0d2"
                        CoProc(NVML) "nvml2"
                    PCIBridge
                      PCI 48:00.0 (SAS)
            PCIBridge
              PCIBridge
                PCIBridge
                  PCI 4b:00.0 (InfiniBand)
                    Net "ibp75s0"
                    OFED "mlx5_2"
                PCIBridge
                  PCIBridge
                    PCIBridge
                      PCI 4e:00.0 (3D)
                        CoProc(CUDA) "cuda3"
                        CoProc(OpenCL) "opencl0d3"
                        CoProc(NVML) "nvml3"
                    PCIBridge
                      PCI 4f:00.0 (SAS)
            PCIBridge
              PCIBridge
                PCIBridge
                  PCI 52:00.0 (NVMExp)
                    Storage(Disk) "nvme4c4n1"
                PCIBridge
                  PCI 53:00.0 (NVMExp)
                    Storage(Disk) "nvme5c5n1"
                PCIBridge
                  PCI 54:00.0 (InfiniBand)
                    Net "ibp84s0"
                    OFED "mlx5_3"
            PCIBridge
              PCI 55:00.0 (Storage)
    Group0 L#2
      NUMANode L#2 (P#2 252GiB)
      L3 L#8 (16MiB)
        L2 L#32 (512KiB) + L1d L#32 (32KiB) + L1i L#32 (32KiB) + Core L#32
          PU L#64 (P#32)
          PU L#65 (P#160)
        L2 L#33 (512KiB) + L1d L#33 (32KiB) + L1i L#33 (32KiB) + Core L#33
          PU L#66 (P#33)
          PU L#67 (P#161)
        L2 L#34 (512KiB) + L1d L#34 (32KiB) + L1i L#34 (32KiB) + Core L#34
          PU L#68 (P#34)
          PU L#69 (P#162)
        L2 L#35 (512KiB) + L1d L#35 (32KiB) + L1i L#35 (32KiB) + Core L#35
          PU L#70 (P#35)
          PU L#71 (P#163)
      L3 L#9 (16MiB)
        L2 L#36 (512KiB) + L1d L#36 (32KiB) + L1i L#36 (32KiB) + Core L#36
          PU L#72 (P#36)
          PU L#73 (P#164)
        L2 L#37 (512KiB) + L1d L#37 (32KiB) + L1i L#37 (32KiB) + Core L#37
          PU L#74 (P#37)
          PU L#75 (P#165)
        L2 L#38 (512KiB) + L1d L#38 (32KiB) + L1i L#38 (32KiB) + Core L#38
          PU L#76 (P#38)
          PU L#77 (P#166)
        L2 L#39 (512KiB) + L1d L#39 (32KiB) + L1i L#39 (32KiB) + Core L#39
          PU L#78 (P#39)
          PU L#79 (P#167)
      L3 L#10 (16MiB)
        L2 L#40 (512KiB) + L1d L#40 (32KiB) + L1i L#40 (32KiB) + Core L#40
          PU L#80 (P#40)
          PU L#81 (P#168)
        L2 L#41 (512KiB) + L1d L#41 (32KiB) + L1i L#41 (32KiB) + Core L#41
          PU L#82 (P#41)
          PU L#83 (P#169)
        L2 L#42 (512KiB) + L1d L#42 (32KiB) + L1i L#42 (32KiB) + Core L#42
          PU L#84 (P#42)
          PU L#85 (P#170)
        L2 L#43 (512KiB) + L1d L#43 (32KiB) + L1i L#43 (32KiB) + Core L#43
          PU L#86 (P#43)
          PU L#87 (P#171)
      L3 L#11 (16MiB)
        L2 L#44 (512KiB) + L1d L#44 (32KiB) + L1i L#44 (32KiB) + Core L#44
          PU L#88 (P#44)
          PU L#89 (P#172)
        L2 L#45 (512KiB) + L1d L#45 (32KiB) + L1i L#45 (32KiB) + Core L#45
          PU L#90 (P#45)
          PU L#91 (P#173)
        L2 L#46 (512KiB) + L1d L#46 (32KiB) + L1i L#46 (32KiB) + Core L#46
          PU L#92 (P#46)
          PU L#93 (P#174)
        L2 L#47 (512KiB) + L1d L#47 (32KiB) + L1i L#47 (32KiB) + Core L#47
          PU L#94 (P#47)
          PU L#95 (P#175)
      HostBridge
        PCIBridge
          PCI 22:00.0 (NVMExp)
            Storage(Disk) "nvme2n1"
        PCIBridge
          PCI 23:00.0 (NVMExp)
            Storage(Disk) "nvme3n1"
    Group0 L#3
      NUMANode L#3 (P#3 252GiB)
      L3 L#12 (16MiB)
        L2 L#48 (512KiB) + L1d L#48 (32KiB) + L1i L#48 (32KiB) + Core L#48
          PU L#96 (P#48)
          PU L#97 (P#176)
        L2 L#49 (512KiB) + L1d L#49 (32KiB) + L1i L#49 (32KiB) + Core L#49
          PU L#98 (P#49)
          PU L#99 (P#177)
        L2 L#50 (512KiB) + L1d L#50 (32KiB) + L1i L#50 (32KiB) + Core L#50
          PU L#100 (P#50)
          PU L#101 (P#178)
        L2 L#51 (512KiB) + L1d L#51 (32KiB) + L1i L#51 (32KiB) + Core L#51
          PU L#102 (P#51)
          PU L#103 (P#179)
      L3 L#13 (16MiB)
        L2 L#52 (512KiB) + L1d L#52 (32KiB) + L1i L#52 (32KiB) + Core L#52
          PU L#104 (P#52)
          PU L#105 (P#180)
        L2 L#53 (512KiB) + L1d L#53 (32KiB) + L1i L#53 (32KiB) + Core L#53
          PU L#106 (P#53)
          PU L#107 (P#181)
        L2 L#54 (512KiB) + L1d L#54 (32KiB) + L1i L#54 (32KiB) + Core L#54
          PU L#108 (P#54)
          PU L#109 (P#182)
        L2 L#55 (512KiB) + L1d L#55 (32KiB) + L1i L#55 (32KiB) + Core L#55
          PU L#110 (P#55)
          PU L#111 (P#183)
      L3 L#14 (16MiB)
        L2 L#56 (512KiB) + L1d L#56 (32KiB) + L1i L#56 (32KiB) + Core L#56
          PU L#112 (P#56)
          PU L#113 (P#184)
        L2 L#57 (512KiB) + L1d L#57 (32KiB) + L1i L#57 (32KiB) + Core L#57
          PU L#114 (P#57)
          PU L#115 (P#185)
        L2 L#58 (512KiB) + L1d L#58 (32KiB) + L1i L#58 (32KiB) + Core L#58
          PU L#116 (P#58)
          PU L#117 (P#186)
        L2 L#59 (512KiB) + L1d L#59 (32KiB) + L1i L#59 (32KiB) + Core L#59
          PU L#118 (P#59)
          PU L#119 (P#187)
      L3 L#15 (16MiB)
        L2 L#60 (512KiB) + L1d L#60 (32KiB) + L1i L#60 (32KiB) + Core L#60
          PU L#120 (P#60)
          PU L#121 (P#188)
        L2 L#61 (512KiB) + L1d L#61 (32KiB) + L1i L#61 (32KiB) + Core L#61
          PU L#122 (P#61)
          PU L#123 (P#189)
        L2 L#62 (512KiB) + L1d L#62 (32KiB) + L1i L#62 (32KiB) + Core L#62
          PU L#124 (P#62)
          PU L#125 (P#190)
        L2 L#63 (512KiB) + L1d L#63 (32KiB) + L1i L#63 (32KiB) + Core L#63
          PU L#126 (P#63)
          PU L#127 (P#191)
      HostBridge
        PCIBridge
          PCIBridge
            PCIBridge
              PCIBridge
                PCIBridge
                  PCIBridge
                    PCIBridge
                      PCI 07:00.0 (3D)
                        CoProc(CUDA) "cuda0"
                        CoProc(OpenCL) "opencl0d0"
                        CoProc(NVML) "nvml0"
                PCIBridge
                  PCI 08:00.0 (NVMExp)
                    Storage(Disk) "nvme0c0n1"
                PCIBridge
                  PCI 09:00.0 (NVMExp)
                    Storage(Disk) "nvme1c1n1"
            PCIBridge
              PCIBridge
                PCIBridge
                  PCI 0c:00.0 (InfiniBand)
                    Net "ibp12s0"
                    OFED "mlx5_0"
                PCIBridge
                  PCIBridge
                    PCIBridge
                      PCI 0f:00.0 (3D)
                        CoProc(CUDA) "cuda1"
                        CoProc(OpenCL) "opencl0d1"
                        CoProc(NVML) "nvml1"
            PCIBridge
              PCIBridge
                PCIBridge
                  PCI 12:00.0 (InfiniBand)
                    Net "ibp18s0"
                    OFED "mlx5_1"
            PCIBridge
              PCI 13:00.0 (Storage)
  Package L#1
    Group0 L#4
      NUMANode L#4 (P#4 252GiB)
      L3 L#16 (16MiB)
        L2 L#64 (512KiB) + L1d L#64 (32KiB) + L1i L#64 (32KiB) + Core L#64
          PU L#128 (P#64)
          PU L#129 (P#192)
        L2 L#65 (512KiB) + L1d L#65 (32KiB) + L1i L#65 (32KiB) + Core L#65
          PU L#130 (P#65)
          PU L#131 (P#193)
        L2 L#66 (512KiB) + L1d L#66 (32KiB) + L1i L#66 (32KiB) + Core L#66
          PU L#132 (P#66)
          PU L#133 (P#194)
        L2 L#67 (512KiB) + L1d L#67 (32KiB) + L1i L#67 (32KiB) + Core L#67
          PU L#134 (P#67)
          PU L#135 (P#195)
      L3 L#17 (16MiB)
        L2 L#68 (512KiB) + L1d L#68 (32KiB) + L1i L#68 (32KiB) + Core L#68
          PU L#136 (P#68)
          PU L#137 (P#196)
        L2 L#69 (512KiB) + L1d L#69 (32KiB) + L1i L#69 (32KiB) + Core L#69
          PU L#138 (P#69)
          PU L#139 (P#197)
        L2 L#70 (512KiB) + L1d L#70 (32KiB) + L1i L#70 (32KiB) + Core L#70
          PU L#140 (P#70)
          PU L#141 (P#198)
        L2 L#71 (512KiB) + L1d L#71 (32KiB) + L1i L#71 (32KiB) + Core L#71
          PU L#142 (P#71)
          PU L#143 (P#199)
      L3 L#18 (16MiB)
        L2 L#72 (512KiB) + L1d L#72 (32KiB) + L1i L#72 (32KiB) + Core L#72
          PU L#144 (P#72)
          PU L#145 (P#200)
        L2 L#73 (512KiB) + L1d L#73 (32KiB) + L1i L#73 (32KiB) + Core L#73
          PU L#146 (P#73)
          PU L#147 (P#201)
        L2 L#74 (512KiB) + L1d L#74 (32KiB) + L1i L#74 (32KiB) + Core L#74
          PU L#148 (P#74)
          PU L#149 (P#202)
        L2 L#75 (512KiB) + L1d L#75 (32KiB) + L1i L#75 (32KiB) + Core L#75
          PU L#150 (P#75)
          PU L#151 (P#203)
      L3 L#19 (16MiB)
        L2 L#76 (512KiB) + L1d L#76 (32KiB) + L1i L#76 (32KiB) + Core L#76
          PU L#152 (P#76)
          PU L#153 (P#204)
        L2 L#77 (512KiB) + L1d L#77 (32KiB) + L1i L#77 (32KiB) + Core L#77
          PU L#154 (P#77)
          PU L#155 (P#205)
        L2 L#78 (512KiB) + L1d L#78 (32KiB) + L1i L#78 (32KiB) + Core L#78
          PU L#156 (P#78)
          PU L#157 (P#206)
        L2 L#79 (512KiB) + L1d L#79 (32KiB) + L1i L#79 (32KiB) + Core L#79
          PU L#158 (P#79)
          PU L#159 (P#207)
      HostBridge
        PCIBridge
          PCI e1:00.0 (Ethernet)
            Net "enp225s0f0"
            OFED "mlx5_10"
          PCI e1:00.1 (Ethernet)
            Net "enp225s0f1"
            OFED "mlx5_11"
        PCIBridge
          PCI e2:00.0 (Ethernet)
            Net "enp226s0"
    Group0 L#5
      NUMANode L#5 (P#5 252GiB)
      L3 L#20 (16MiB)
        L2 L#80 (512KiB) + L1d L#80 (32KiB) + L1i L#80 (32KiB) + Core L#80
          PU L#160 (P#80)
          PU L#161 (P#208)
        L2 L#81 (512KiB) + L1d L#81 (32KiB) + L1i L#81 (32KiB) + Core L#81
          PU L#162 (P#81)
          PU L#163 (P#209)
        L2 L#82 (512KiB) + L1d L#82 (32KiB) + L1i L#82 (32KiB) + Core L#82
          PU L#164 (P#82)
          PU L#165 (P#210)
        L2 L#83 (512KiB) + L1d L#83 (32KiB) + L1i L#83 (32KiB) + Core L#83
          PU L#166 (P#83)
          PU L#167 (P#211)
      L3 L#21 (16MiB)
        L2 L#84 (512KiB) + L1d L#84 (32KiB) + L1i L#84 (32KiB) + Core L#84
          PU L#168 (P#84)
          PU L#169 (P#212)
        L2 L#85 (512KiB) + L1d L#85 (32KiB) + L1i L#85 (32KiB) + Core L#85
          PU L#170 (P#85)
          PU L#171 (P#213)
        L2 L#86 (512KiB) + L1d L#86 (32KiB) + L1i L#86 (32KiB) + Core L#86
          PU L#172 (P#86)
          PU L#173 (P#214)
        L2 L#87 (512KiB) + L1d L#87 (32KiB) + L1i L#87 (32KiB) + Core L#87
          PU L#174 (P#87)
          PU L#175 (P#215)
      L3 L#22 (16MiB)
        L2 L#88 (512KiB) + L1d L#88 (32KiB) + L1i L#88 (32KiB) + Core L#88
          PU L#176 (P#88)
          PU L#177 (P#216)
        L2 L#89 (512KiB) + L1d L#89 (32KiB) + L1i L#89 (32KiB) + Core L#89
          PU L#178 (P#89)
          PU L#179 (P#217)
        L2 L#90 (512KiB) + L1d L#90 (32KiB) + L1i L#90 (32KiB) + Core L#90
          PU L#180 (P#90)
          PU L#181 (P#218)
        L2 L#91 (512KiB) + L1d L#91 (32KiB) + L1i L#91 (32KiB) + Core L#91
          PU L#182 (P#91)
          PU L#183 (P#219)
      L3 L#23 (16MiB)
        L2 L#92 (512KiB) + L1d L#92 (32KiB) + L1i L#92 (32KiB) + Core L#92
          PU L#184 (P#92)
          PU L#185 (P#220)
        L2 L#93 (512KiB) + L1d L#93 (32KiB) + L1i L#93 (32KiB) + Core L#93
          PU L#186 (P#93)
          PU L#187 (P#221)
        L2 L#94 (512KiB) + L1d L#94 (32KiB) + L1i L#94 (32KiB) + Core L#94
          PU L#188 (P#94)
          PU L#189 (P#222)
        L2 L#95 (512KiB) + L1d L#95 (32KiB) + L1i L#95 (32KiB) + Core L#95
          PU L#190 (P#95)
          PU L#191 (P#223)
      HostBridge
        PCIBridge
          PCIBridge
            PCIBridge
              PCIBridge
                PCIBridge
                  PCIBridge
                    PCIBridge
                      PCI b7:00.0 (3D)
                        CoProc(CUDA) "cuda6"
                        CoProc(OpenCL) "opencl0d6"
                        CoProc(NVML) "nvml6"
            PCIBridge
              PCIBridge
                PCIBridge
                  PCI ba:00.0 (InfiniBand)
                    Net "ibp186s0"
                    OFED "mlx5_8"
                PCIBridge
                  PCIBridge
                    PCIBridge
                      PCI bd:00.0 (3D)
                        CoProc(CUDA) "cuda7"
                        CoProc(OpenCL) "opencl0d7"
                        CoProc(NVML) "nvml7"
            PCIBridge
              PCIBridge
                PCIBridge
                  PCIBridge
                    PCIBridge
                      PCIBridge
                        PCIBridge
                          PCI(NVSwitch) c4:00.0 (Bridge)
                        PCIBridge
                          PCI(NVSwitch) c5:00.0 (Bridge)
                        PCIBridge
                          PCI(NVSwitch) c6:00.0 (Bridge)
                        PCIBridge
                          PCI(NVSwitch) c7:00.0 (Bridge)
                        PCIBridge
                          PCI(NVSwitch) c8:00.0 (Bridge)
                        PCIBridge
                          PCI(NVSwitch) c9:00.0 (Bridge)
                PCIBridge
                  PCI ca:00.0 (NVMExp)
                    Storage(Disk) "nvme8c8n1"
                PCIBridge
                  PCI cb:00.0 (NVMExp)
                    Storage(Disk) "nvme9c9n1"
                PCIBridge
                  PCI cc:00.0 (InfiniBand)
                    Net "ibp204s0"
                    OFED "mlx5_9"
            PCIBridge
              PCI cd:00.0 (Storage)
    Group0 L#6
      NUMANode L#6 (P#6 252GiB)
      L3 L#24 (16MiB)
        L2 L#96 (512KiB) + L1d L#96 (32KiB) + L1i L#96 (32KiB) + Core L#96
          PU L#192 (P#96)
          PU L#193 (P#224)
        L2 L#97 (512KiB) + L1d L#97 (32KiB) + L1i L#97 (32KiB) + Core L#97
          PU L#194 (P#97)
          PU L#195 (P#225)
        L2 L#98 (512KiB) + L1d L#98 (32KiB) + L1i L#98 (32KiB) + Core L#98
          PU L#196 (P#98)
          PU L#197 (P#226)
        L2 L#99 (512KiB) + L1d L#99 (32KiB) + L1i L#99 (32KiB) + Core L#99
          PU L#198 (P#99)
          PU L#199 (P#227)
      L3 L#25 (16MiB)
        L2 L#100 (512KiB) + L1d L#100 (32KiB) + L1i L#100 (32KiB) + Core L#100
          PU L#200 (P#100)
          PU L#201 (P#228)
        L2 L#101 (512KiB) + L1d L#101 (32KiB) + L1i L#101 (32KiB) + Core L#101
          PU L#202 (P#101)
          PU L#203 (P#229)
        L2 L#102 (512KiB) + L1d L#102 (32KiB) + L1i L#102 (32KiB) + Core L#102
          PU L#204 (P#102)
          PU L#205 (P#230)
        L2 L#103 (512KiB) + L1d L#103 (32KiB) + L1i L#103 (32KiB) + Core L#103
          PU L#206 (P#103)
          PU L#207 (P#231)
      L3 L#26 (16MiB)
        L2 L#104 (512KiB) + L1d L#104 (32KiB) + L1i L#104 (32KiB) + Core L#104
          PU L#208 (P#104)
          PU L#209 (P#232)
        L2 L#105 (512KiB) + L1d L#105 (32KiB) + L1i L#105 (32KiB) + Core L#105
          PU L#210 (P#105)
          PU L#211 (P#233)
        L2 L#106 (512KiB) + L1d L#106 (32KiB) + L1i L#106 (32KiB) + Core L#106
          PU L#212 (P#106)
          PU L#213 (P#234)
        L2 L#107 (512KiB) + L1d L#107 (32KiB) + L1i L#107 (32KiB) + Core L#107
          PU L#214 (P#107)
          PU L#215 (P#235)
      L3 L#27 (16MiB)
        L2 L#108 (512KiB) + L1d L#108 (32KiB) + L1i L#108 (32KiB) + Core L#108
          PU L#216 (P#108)
          PU L#217 (P#236)
        L2 L#109 (512KiB) + L1d L#109 (32KiB) + L1i L#109 (32KiB) + Core L#109
          PU L#218 (P#109)
          PU L#219 (P#237)
        L2 L#110 (512KiB) + L1d L#110 (32KiB) + L1i L#110 (32KiB) + Core L#110
          PU L#220 (P#110)
          PU L#221 (P#238)
        L2 L#111 (512KiB) + L1d L#111 (32KiB) + L1i L#111 (32KiB) + Core L#111
          PU L#222 (P#111)
          PU L#223 (P#239)
    Group0 L#7
      NUMANode L#7 (P#7 252GiB)
      L3 L#28 (16MiB)
        L2 L#112 (512KiB) + L1d L#112 (32KiB) + L1i L#112 (32KiB) + Core L#112
          PU L#224 (P#112)
          PU L#225 (P#240)
        L2 L#113 (512KiB) + L1d L#113 (32KiB) + L1i L#113 (32KiB) + Core L#113
          PU L#226 (P#113)
          PU L#227 (P#241)
        L2 L#114 (512KiB) + L1d L#114 (32KiB) + L1i L#114 (32KiB) + Core L#114
          PU L#228 (P#114)
          PU L#229 (P#242)
        L2 L#115 (512KiB) + L1d L#115 (32KiB) + L1i L#115 (32KiB) + Core L#115
          PU L#230 (P#115)
          PU L#231 (P#243)
      L3 L#29 (16MiB)
        L2 L#116 (512KiB) + L1d L#116 (32KiB) + L1i L#116 (32KiB) + Core L#116
          PU L#232 (P#116)
          PU L#233 (P#244)
        L2 L#117 (512KiB) + L1d L#117 (32KiB) + L1i L#117 (32KiB) + Core L#117
          PU L#234 (P#117)
          PU L#235 (P#245)
        L2 L#118 (512KiB) + L1d L#118 (32KiB) + L1i L#118 (32KiB) + Core L#118
          PU L#236 (P#118)
          PU L#237 (P#246)
        L2 L#119 (512KiB) + L1d L#119 (32KiB) + L1i L#119 (32KiB) + Core L#119
          PU L#238 (P#119)
          PU L#239 (P#247)
      L3 L#30 (16MiB)
        L2 L#120 (512KiB) + L1d L#120 (32KiB) + L1i L#120 (32KiB) + Core L#120
          PU L#240 (P#120)
          PU L#241 (P#248)
        L2 L#121 (512KiB) + L1d L#121 (32KiB) + L1i L#121 (32KiB) + Core L#121
          PU L#242 (P#121)
          PU L#243 (P#249)
        L2 L#122 (512KiB) + L1d L#122 (32KiB) + L1i L#122 (32KiB) + Core L#122
          PU L#244 (P#122)
          PU L#245 (P#250)
        L2 L#123 (512KiB) + L1d L#123 (32KiB) + L1i L#123 (32KiB) + Core L#123
          PU L#246 (P#123)
          PU L#247 (P#251)
      L3 L#31 (16MiB)
        L2 L#124 (512KiB) + L1d L#124 (32KiB) + L1i L#124 (32KiB) + Core L#124
          PU L#248 (P#124)
          PU L#249 (P#252)
        L2 L#125 (512KiB) + L1d L#125 (32KiB) + L1i L#125 (32KiB) + Core L#125
          PU L#250 (P#125)
          PU L#251 (P#253)
        L2 L#126 (512KiB) + L1d L#126 (32KiB) + L1i L#126 (32KiB) + Core L#126
          PU L#252 (P#126)
          PU L#253 (P#254)
        L2 L#127 (512KiB) + L1d L#127 (32KiB) + L1i L#127 (32KiB) + Core L#127 + PU L#254 (P#127)
      HostBridge
        PCIBridge
          PCIBridge
            PCIBridge
              PCIBridge
                PCIBridge
                  PCIBridge
                    PCIBridge
                      PCI 87:00.0 (3D)
                        CoProc(CUDA) "cuda4"
                        CoProc(OpenCL) "opencl0d4"
                        CoProc(NVML) "nvml4"
                    PCIBridge
                      PCI 88:00.0 (SAS)
                PCIBridge
                  PCI 89:00.0 (NVMExp)
                    Storage(Disk) "nvme6c6n1"
                PCIBridge
                  PCI 8a:00.0 (NVMExp)
                    Storage(Disk) "nvme7c7n1"
            PCIBridge
              PCIBridge
                PCIBridge
                  PCI 8d:00.0 (InfiniBand)
                    Net "ibp141s0"
                    OFED "mlx5_6"
                PCIBridge
                  PCIBridge
                    PCIBridge
                      PCI 90:00.0 (3D)
                        CoProc(CUDA) "cuda5"
                        CoProc(OpenCL) "opencl0d5"
                        CoProc(NVML) "nvml5"
                    PCIBridge
                      PCI 91:00.0 (SAS)
            PCIBridge
              PCIBridge
                PCIBridge
                  PCI 94:00.0 (InfiniBand)
                    Net "ibp148s0"
                    OFED "mlx5_7"
            PCIBridge
              PCI 95:00.0 (Storage)

8，总结

8.1 一键编译 hwloc

git clone https://github.com/open-mpi/hwloc.git &&\
cd hwloc &&\
git checkout hwloc-2.9.3 &&\
./autogen.sh &&\
mkdir build &&\
cd build &&\
../configure --prefix=/home/hipper/ex_hwloc/tmp/localhw &&\
make -j &&\
make install