eBPF(一）samples/bpf 交叉编译

伴生_0904

已于 2024-08-29 14:30:41 修改

阅读量679

点赞数 12

分类专栏： eBPF 文章标签： eBPF

于 2024-08-29 03:26:07 首次发布

本文链接：https://blog.csdn.net/weixin_45705773/article/details/141653056

版权

eBPF 专栏收录该内容

1 篇文章 0 订阅

订阅专栏

编译环境

宿主机： 5.15.0-60-generic #66-Ubuntu SMP Fri Jan 20 14:29:49 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
目标主机：Linux tegra-ubuntu 5.15.98-rt-tegra #1 SMP PREEMPT_RT Thu Jun 27 18:01:52 CST 2024 aarch64 aarch64 aarch64 GNU/Linux

下载内核源码地址：linux进程调度（1）内核目录介绍-CSDN博客

LIBBPF API — libbpf documentation

编译步骤

make  ARCH=arm64  CROSS_COMPILE=/xxx_path/aarch64-linux-gnu-  defconfig
make  ARCH=arm64  CROSS_COMPILE=/xxx_path/aarch64-linux-gnu-  menuconfig
make  ARCH=arm64  CROSS_COMPILE=/xxx_path/aarch64-linux-gnu-  prepare
make  ARCH=arm64  CROSS_COMPILE=/xxx_path/aarch64-linux-gnu-  -j8
make  ARCH=arm64  CROSS_COMPILE=/xxx_path/aarch64-linux-gnu-  headers_install
make  ARCH=arm64  CROSS_COMPILE=/xxx_path/aarch64-linux-gnu-  M=samples/bpf

编译过程

1. make ARCH=arm64 CROSS_COMPILE=/xxx_path/aarch64-linux-gnu- defconfig

使用defconfig生成 .config，这里的defconfig, 因为指定了arm64, 最终使用的是 arch/arm64/configs/defconfig, 详细可看Makefile实现

# make  ARCH=arm64  CROSS_COMPILE=/xxx_path/aarch64-linux-gnu-  defconfig
  HOSTCC  scripts/basic/fixdep
  HOSTCC  scripts/kconfig/conf.o
  HOSTCC  scripts/kconfig/confdata.o
  HOSTCC  scripts/kconfig/expr.o
  LEX     scripts/kconfig/lexer.lex.c
  YACC    scripts/kconfig/parser.tab.[ch]
  HOSTCC  scripts/kconfig/lexer.lex.o
  HOSTCC  scripts/kconfig/menu.o
  HOSTCC  scripts/kconfig/parser.tab.o
  HOSTCC  scripts/kconfig/preprocess.o
  HOSTCC  scripts/kconfig/symbol.o
  HOSTCC  scripts/kconfig/util.o
  HOSTLD  scripts/kconfig/conf
*** Default configuration is based on 'defconfig'
#
# configuration written to .config
#

2. make ARCH=arm64 CROSS_COMPILE=/xxx_path/aarch64-linux-gnu- menuconfig

配置ebpf依赖的内核选项

1)bpf功能打开

2)网络子模块选项，BPF支持

3）开启kernel debug info，同时支持 BTF （BPF通过BTF实现编译一次，到处运行）

4)开启trace功能，按需设置

3. make ARCH=arm64 CROSS_COMPILE=/xxx_path/aarch64-linux-gnu- prepare

这一步开始容易报错，主要是缺少工具或者库，

报错1：缺少bc工具，apt install 安装，bc是编译环境需要，不是ebpf链接需要，所以安装宿主机版本就行

/bin/sh: 1: bc: not found
apt install bc

报错2：缺少libelf.h，这一步踩了两个坑，1）以为是目标编译依赖libelf，交叉编译了libelf库，还是一样报错，2）调试makefile，发现到这一步时，CC从aarch64-linux-gnu-gcc 变成了宿主机gcc，以为makefile有问题，折腾了许久弄明白，过程中依赖一些宿主机工具，比如bpftool工具，用于从vmlinux中提取btf生成vmlinux.h, CC会切换

正解是安装宿主机版本libelf-dev

libbpf.c:47:10: fatal error: libelf.h: No such file or directory
   47 | #include <libelf.h>

$apt-get install -f libelf-dev

4. make ARCH=arm64 CROSS_COMPILE=/xxx_path/aarch64-linux-gnu- -j8

编译vmlinux，编译samples/bpf依赖从vmlinux里面提取BTF

make ARCH=arm64 CROSS_COMPILE=/xxx_path/aarch64-linux-gnu- vmlinux -j8

报错缺少pahole ，pahole 在dwarves中，apt install dwarves 重新再编译成功

BTF: .tmp_vmlinux.btf: pahole (pahole) is not available
$ apt install dwarves

5. make ARCH=arm64 CROSS_COMPILE=/xxx_path/aarch64-linux-gnu- headers_install

安装头文件，这一步无异常

6. make ARCH=arm64 CROSS_COMPILE=/xxx_path/aarch64-linux-gnu- M=samples/bpf

1)再次报错缺少libelf.h，这一步缺少的是目标依赖的libelf，需要交叉编译libelf, 前面交叉编译的libelf刚好派上用场，安装libelf到交叉编译链后，再次编译解决，其他缺少的目标编译依赖库一样操作

  libbpf.c:47:10: fatal error: libelf.h: No such file or directory
   47 | #include <libelf.h>

2)继续编译，再次报错，缺少LLVM，安装新版本llvm-12，默认会安装llvm-6编译有问题不细究

*** ERROR: Cannot find LLVM tool llc

1.添加 LLVM 官方存储库：
sudo apt install wget
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
2.添加版本信息（以 12.x 为例，如果更高版本可选，可以相应修改）：
echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-12 main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-focal-12.list
3.更新包列表：
sudo apt update
sudo apt install llvm-12
apt install llvm-12
4.创建软连接，缺啥创建啥
ln -s /usr/bin/llc-12 /usr/bin/llc
ln -s /usr/bin/llvm-strip-12 /usr/bin/llvm-strip
ln -s /usr/bin/opt-1 /usr/bin/llvm-strip
ln -s /usr/bin/opt-12 /usr/bin/opt
ln -s /usr/bin/llvm-dis-12 /usr/bin/llvm-dis

3)再次报错bpftool Exec format error，使用file命令看bpftool工具是arm64的，查看makefile，正是前面提到的从vmlinux提取BTF等信息生成vmlinux.h，这明显需要运行宿主机上x86 bpftool，为什么makefile里面就变成了用arm64版本，回头再研究

/bin/sh: 1: /mnt/data/share/kernel/new/linux-5.15.98/samples/bpf/bpftool/bpftool: Exec format error
make[1]: *** [samples/bpf/Makefile:372: samples/bpf/vmlinux.h] Error 2
make[1]: *** Deleting file 'samples/bpf/vmlinux.h'
make: *** [Makefile:1903: samples/bpf] Error 2

4)继续往下，搜索bpftool，发现有5个，试了下./samples/bpf/bpftool/bootstrap/bpftool 是x86版本

 find ./ -name bpftool
./samples/bpf/bpftool
./samples/bpf/bpftool/bpftool
./samples/bpf/bpftool/bootstrap/bpftool
./tools/bpf/bpftool
./tools/bpf/bpftool/bash-completion/bpftool

5)修改makefile，把BPFTOOL换成x86版本，这次报另外一个错误，xdp相关的例子编译不过

libbpf: map 'rx_cnt': unexpected def kind var.
Error: failed to open BPF object file: Invalid argument
make[1]: *** [samples/bpf/Makefile:430: samples/bpf/xdp_monitor.skel.h] Error 255
make[1]: *** Deleting file 'samples/bpf/xdp_monitor.skel.h'
make: *** [Makefile:1903: samples/bpf] Error 2

6)先不研究，干脆利索，把makefile中xdp相关的例子全去掉，再次编译，竟然成功，进到samples/bpf，xdp外的例子都编译成功

7)samples/bpf 拷贝到目标主机运行./trace_output报错找不到map信息

$ ./trace_output
ERROR: finding a map in obj file failed

8)查看trace_output_kern.c，map映射的是kprobe, 目标主机确认没打开kprobe

9)修改trace_output_kern.c， trace_output_user.c，改为测试sched_swich tracepoint，目标主机常规的tracepoint都有

#include <linux/ptrace.h>
#include <linux/version.h>
#include <uapi/linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "trace_common.h"

struct {
    __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
    __uint(key_size, sizeof(int));
    __uint(value_size, sizeof(u32));
    __uint(max_entries, 2);
} my_map SEC(".maps");

// 使用 tracepoint 追踪进程切换
// 使用 void 作为参数表示
SEC("tracepoint/sched/sched_switch")
int bpf_prog1(void *ctx)
{
    struct {
        u64 pid;
        u64 cookie;
    } data;

    data.pid = bpf_get_current_pid_tgid(); // 获取当前pid
    data.cookie = 0x12345678;  // 设定 cookie

    // 将数据发送到 perf buffer
    bpf_perf_event_output(ctx, &my_map, BPF_F_CURRENT_CPU, &data, sizeof(data));

    return 0;
}
char _license[] SEC("license") = "GPL";
u32 _version SEC("version") = LINUX_VERSION_CODE;

// SPDX-License-Identifier: GPL-2.0-only
#include <stdio.h>
#include <fcntl.h>
#include <poll.h>
#include <time.h>
#include <signal.h>
#include <bpf/libbpf.h>

static __u64 time_get_ns(void)
{
    struct timespec ts;

    clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec * 1000000000ull + ts.tv_nsec;
}

static __u64 start_time;
static __u64 cnt;

#define MAX_CNT 100000ll

static void print_bpf_output(void *ctx, int cpu, void *data, __u32 size)
{
    struct {
        __u64 pid;
        __u64 cookie;
    } *e = data;

    if (e->cookie != 0x12345678) {
        printf("BUG pid %llx cookie %llx sized %d\n",
               e->pid, e->cookie, size);
        return;
    }

    cnt++;

    if (cnt == MAX_CNT) {
        printf("recv %lld events per sec\n",
               MAX_CNT * 1000000000ll / (time_get_ns() - start_time));
        return;
    }
}

int main(int argc, char **argv)
{
    struct perf_buffer_opts pb_opts = {};
    struct bpf_link *link = NULL;
    struct bpf_program *prog;
    struct perf_buffer *pb;
    struct bpf_object *obj;
    int map_fd, ret = 0;
    char filename[256];
    FILE *f;

    snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
    obj = bpf_object__open_file(filename, NULL);
    if (libbpf_get_error(obj)) {
        fprintf(stderr, "ERROR: opening BPF object file failed\n");
        return 0;
    }

    // load BPF program
    if (bpf_object__load(obj)) {
        fprintf(stderr, "ERROR: loading BPF object file failed\n");
        goto cleanup;
    }

    // 获取 map fd
    map_fd = bpf_object__find_map_fd_by_name(obj, "my_map");
    if (map_fd < 0) {
        fprintf(stderr, "ERROR: finding a map in obj file failed\n");
        goto cleanup;
    }

    // 找到 BPF program
    prog = bpf_object__find_program_by_name(obj, "bpf_prog1");
    if (libbpf_get_error(prog)) {
        fprintf(stderr, "ERROR: finding a prog in obj file failed\n");
        goto cleanup;
    }

    // 附加程序
    link = bpf_program__attach(prog);
    if (libbpf_get_error(link)) {
        fprintf(stderr, "ERROR: bpf_program__attach failed\n");
        link = NULL;
        goto cleanup;
    }

    pb_opts.sample_cb = print_bpf_output;
    pb = perf_buffer__new(map_fd, 8, &pb_opts);
    ret = libbpf_get_error(pb);
    if (ret) {
        printf("failed to setup perf_buffer: %d\n", ret);
        return 1;
    }

    f = popen("taskset 1 dd if=/dev/zero of=/dev/null", "r");
    (void) f;

    start_time = time_get_ns();
    while ((ret = perf_buffer__poll(pb, 1000)) >= 0 && cnt < MAX_CNT) {
    }
    kill(0, SIGINT);

cleanup:
    bpf_link__destroy(link);
    bpf_object__close(obj);
    return ret;
}

10)成功运行

$ ./trace_output
recv 577113 events per sec
2600+0 records in
2599+0 records out
1330688 bytes (1.3 MB, 1.3 MiB) copied, 0.106804 s, 12.5 MB/s

伴生_0904

关注

12
点赞
踩
9

收藏

觉得还不错? 一键收藏
0
评论
复制链接

分享到 QQ

分享到新浪微博

扫一扫

专栏目录