Containerd 安装过程以及踩的坑

安装过程

由于K8S在后续版本中将停止对docker的支持，因此最近摸鱼的时候想到了研究下怎么用containerd事情，在尝试过程中碰到了一些问题，特此将相关过程记录，以作备忘。

containerd安装很简单，访问项目GitHub，在release中找到要安装的版本和操作系统对应的二进制包，下载解压即可得到所有需要的二进制文件。

以centos下安装1.4.6版本containerd为例

wget https://github.com/containerd/containerd/releases/download/v1.4.6/containerd-1.4.6-linux-amd64.tar.gz

解压后得到一个bin目录，里面包含了如下文件：

[root@localhost ~]# ls bin/
containerd  containerd-shim  containerd-shim-runc-v1  containerd-shim-runc-v2  ctr

解释下这几个文件的作用：

containerd：containerd程序主体

containerd-shim-XXX：containerd的进程，用于操作runc

ctr：containerd自带的命令行工具

将这个目录加入到你的PATH目录下，就可以直接在终端执行了。

下面要初始化containerd的配置，containerd启动时默认会读取/etc/containerd/config.toml文件中的配置，可以通过启动参数--config修改，不过我们就不那么麻烦了，直接默认配置即可。

先创建配置文件的目录：

mkdir -p /etc/containerd/

再初始化配置：

containerd config default> /etc/containerd/config.toml

现在一个配置文件模板/etc/containerd/config.toml就生成了。理论上正常情况下，此时直接启动也是可以的，但鉴于之前碰到的一些坑，所以还是有必要介绍一下配置文件中的一些配置项。鉴于这部分资料是真的少，而且官方文档也几乎没有说明，因此这真的是撞破头得出来的经验。。。

首先看一下这个配置文件长什么样：

version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
plugin_dir = ""
disabled_plugins = []
required_plugins = []
oom_score = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  tcp_address = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216

[ttrpc]
  address = ""
  uid = 0
  gid = 0

[debug]
  address = ""
  uid = 0
  gid = 0
  level = ""

[metrics]
  address = ""
  grpc_histogram = false

[cgroup]
  path = ""

[timeouts]
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[plugins]
  [plugins."io.containerd.gc.v1.scheduler"]
    pause_threshold = 0.02
    deletion_threshold = 0
    mutation_threshold = 100
    schedule_delay = "0s"
    startup_delay = "100ms"
  [plugins."io.containerd.grpc.v1.cri"]
    disable_tcp_service = true
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    stream_idle_timeout = "4h0m0s"
    enable_selinux = false
    selinux_category_range = 1024
    sandbox_image = "k8s.gcr.io/pause:3.2"
    stats_collect_period = 10
    systemd_cgroup = false
    enable_tls_streaming = false
    max_container_log_line_size = 16384
    disable_cgroup = false
    disable_apparmor = false
    restrict_oom_score_adj = false
    max_concurrent_downloads = 3
    disable_proc_mount = false
    unset_seccomp_profile = ""
    tolerate_missing_hugetlb_controller = true
    disable_hugetlb_controller = true
    ignore_image_defined_volumes = false
    [plugins."io.containerd.grpc.v1.cri".containerd]
      snapshotter = "overlayfs"
      default_runtime_name = "runc"
      no_pivot = false
      disable_snapshot_annotations = true
      discard_unpacked_layers = false
      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
        base_runtime_spec = ""
      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
        base_runtime_spec = ""
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          runtime_type = "io.containerd.runc.v2"
          runtime_engine = ""
          runtime_root = ""
          privileged_without_host_devices = false
          base_runtime_spec = ""
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
      max_conf_num = 1
      conf_template = ""
    [plugins."io.containerd.grpc.v1.cri".registry]
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://registry-1.docker.io"]
    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = ""
    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""
  [plugins."io.containerd.internal.v1.opt"]
    path = "/opt/containerd"
  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"
  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"
  [plugins."io.containerd.monitor.v1.cgroups"]
    no_prometheus = false
  [plugins."io.containerd.runtime.v1.linux"]
    shim = "containerd-shim"
    runtime = "runc"
    runtime_root = ""
    no_shim = false
    shim_debug = false
  [plugins."io.containerd.runtime.v2.task"]
    platforms = ["linux/amd64"]
  [plugins."io.containerd.service.v1.diff-service"]
    default = ["walking"]
  [plugins."io.containerd.snapshotter.v1.devmapper"]
    root_path = "/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs"
    pool_name = ""
    base_image_size = ""
    async_remove = false

可以看到这个配置文件很长，看起来很恐怖，尤其是在没有任何文档和注释说明的情况下。大部分不太用管，介绍一下比较关键的几个配置，以及碰到的问题：

root = "/var/lib/containerd"

该目录用于containerd管理镜像的一系列目录，根据目录名可以区分用途。目录结构如下：

[root@localhost containerd]# ll
总用量 36
drwxr-xr-x 4 root root 4096 7月   2 11:57 io.containerd.content.v1.content
drwx------ 2 root root 4096 7月   5 11:41 io.containerd.grpc.v1.introspection
drwx--x--x 2 root root 4096 7月   2 11:41 io.containerd.metadata.v1.bolt
drwx--x--x 2 root root 4096 7月   2 11:41 io.containerd.runtime.v1.linux
drwx--x--x 3 root root 4096 7月   2 15:11 io.containerd.runtime.v2.task
drwxr-xr-x 2 root root 4096 7月   2 11:41 io.containerd.snapshotter.v1.btrfs
drwx------ 3 root root 4096 7月   2 11:41 io.containerd.snapshotter.v1.native
drwx------ 3 root root 4096 7月   2 11:57 io.containerd.snapshotter.v1.overlayfs
drwx------ 2 root root 4096 7月   5 15:17 tmpmounts

其中io.containerd.snapshotter.v1.overlayfs目录就是用于overlayfs下存储snapshot的，其他目录以此类推。

此处可见坑1

state = "/run/containerd"

从snapshot中创建的rootfs、config.json等一系列文件。用于创建容器后的非持久化存储。目录结构如下：

-rw-r--r-- 1 root root   89 7月   5 15:21 address
-rw-r--r-- 1 root root 3259 7月   5 15:21 config.json
-rw-r--r-- 1 root root    5 7月   5 15:21 init.pid
prwx------ 1 root root    0 7月   5 15:21 log
-rw-r--r-- 1 root root    0 7月   5 15:21 log.json
-rw------- 1 root root    2 7月   5 15:21 options.json
drwxr-xr-x 1 root root 4096 7月   5 15:21 rootfs
-rw------- 1 root root    0 7月   5 15:21 runtime
lrwxrwxrwx 1 root root   63 7月   5 15:21 work -> /mnt/containerd/io.containerd.runtime.v2.task/default/entry-mch

如果在容器根目录中创建了一个临时文件test.txt，可以在rootfs下找到。这个目录会随着容器的销毁而销毁。

snapshotter = "overlayfs"

snapshot模块，默认为overlayfs

        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]

          endpoint = ["https://registry-1.docker.io"]

镜像仓库配置，可以如法炮制增加自己的私有镜像仓库：

        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."172.1.1.16:8083"]

          endpoint = ["http://172.1.1.16:8083"]

若你的仓库是非安全的http，则需要增加如下配置：

      [plugins."io.containerd.grpc.v1.cri".registry.configs]

        [plugins."io.containerd.grpc.v1.cri".registry.configs."172.1.1.16:8083".tls]

          insecure_skip_verify = true

https则需要配置一系列认证，此处省略（doge）

在使用ctr工具拉镜像，非安全仓库可能会出现坑2

基本上比较关键的配置就是这些。在修改完毕后，就可以配置system units了，官方示例如下：

[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target
[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target

After=network.target 一定要配置，

ExecStartPre=-/sbin/modprobe overlay 要与配置文件中snapshot的模块一致。

ExecStart=/usr/local/bin/containerd containerd二进制文件所在目录，若配置文件目录不为默认，则需手动加参数。

完成后执行systemctl start containerd即可运行。

注意，此时还没有安装runc，是无法启动容器的。runc的安装非常简单，下载二进制文件后放到PATH下即可。注意，此时可能会遇到坑3

坑1

root目录使用xfs文件系统时，会出现无法创建snapshot的问题，报错如下：

● containerd.service - containerd container runtime
   Loaded: loaded (/usr/lib/systemd/system/containerd.service; disabled; vendor preset: disabled)
   Active: inactive (dead)
     Docs: https://containerd.io

7月 02 10:30:59 localhost.localdomain containerd[2617]: time="2021-07-02T10:30:59.267658802+08:00" level=info msg="loading plugin \"io.containerd.grpc.v1.cri\"..." type=io.containerd.grpc.v1
7月 02 10:30:59 localhost.localdomain containerd[2617]: time="2021-07-02T10:30:59.268039537+08:00" level=info msg="Start cri plugin with config {PluginConfig:{ContainerdConfig:{Snapshotter:overlayfs DefaultRuntimeName:runc DefaultRuntime:{Type: Engine: PodAnnotations:[] ContainerAnnotations:[] Root: Options:<nil> PrivilegedWithoutHostDevices:false BaseRuntimeSpec:} UntrustedWorkloadRuntime:{Type: Engine: PodAnnotations:[] ContainerAnnotations:[] Root: Options:<nil> PrivilegedWithoutHostDevices:false BaseRuntimeSpec:} Runtimes:map[runc:{Type:io.containerd.runc.v2 Engine: PodAnnotations:[] ContainerAnnotations:[] Root: Options:0xc00041b860 PrivilegedWithoutHostDevices:false BaseRuntimeSpec:}] NoPivot:false DisableSnapshotAnnotations:true DiscardUnpackedLayers:false} CniConfig:{NetworkPluginBinDir:/opt/cni/bin NetworkPluginConfDir:/etc/cni/net.d NetworkPluginMaxConfNum:1 NetworkPluginConfTemplate:} Registry:{Mirrors:map[172.18.121.16:8083:{Endpoints:[http://172.18.121.16:8083]} docker.io:{Endpoints:[https://registry-1.docker.io]}] Configs:map[] Auths:map[] Headers:map[]} ImageDecryption:{KeyModel:} DisableTCPService:true StreamServerAddress:127.0.0.1 StreamServerPort:0 StreamIdleTimeout:4h0m0s EnableSelinux:false SelinuxCategoryRange:1024 SandboxImage:k8s.gcr.io/pause:3.2 StatsCollectPeriod:10 SystemdCgroup:false EnableTLSStreaming:false X509KeyPairStreaming:{TLSCertFile: TLSKeyFile:} MaxContainerLogLineSize:16384 DisableCgroup:false DisableApparmor:false RestrictOOMScoreAdj:false MaxConcurrentDownloads:3 DisableProcMount:false UnsetSeccompProfile: TolerateMissingHugetlbController:true DisableHugetlbController:true IgnoreImageDefinedVolumes:false} ContainerdRootDir:/var/lib/containerd ContainerdEndpoint:/run/containerd/containerd.sock RootDir:/var/lib/containerd/io.containerd.grpc.v1.cri StateDir:/run/containerd/io.containerd.grpc.v1.cri}"
7月 02 10:30:59 localhost.localdomain containerd[2617]: time="2021-07-02T10:30:59.268115703+08:00" level=info msg="Connect containerd service"
7月 02 10:30:59 localhost.localdomain containerd[2617]: time="2021-07-02T10:30:59.268199993+08:00" level=warning msg="failed to load plugin io.containerd.grpc.v1.cri" error="failed to create CRI service: failed to find snapshotter \"overlayfs\""
7月 02 10:30:59 localhost.localdomain containerd[2617]: time="2021-07-02T10:30:59.268219975+08:00" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." type=io.containerd.grpc.v1
7月 02 10:30:59 localhost.localdomain containerd[2617]: time="2021-07-02T10:30:59.268610460+08:00" level=info msg=serving... address=/run/containerd/containerd.sock.ttrpc
7月 02 10:30:59 localhost.localdomain containerd[2617]: time="2021-07-02T10:30:59.268717460+08:00" level=info msg=serving... address=/run/containerd/containerd.sock
7月 02 10:30:59 localhost.localdomain containerd[2617]: time="2021-07-02T10:30:59.268752374+08:00" level=info msg="containerd successfully booted in 0.039053s"
7月 02 11:02:37 localhost.localdomain systemd[1]: Stopping containerd container runtime...
7月 02 11:02:37 localhost.localdomain systemd[1]: Stopped containerd container runtime.

估计是由于xfs不支持overlayfs导致。更换ext4文件系统解决。

坑2

ctr命令从非安全仓库拉取镜像时，出现如下错误：

 http: server gave HTTP response to HTTPS client

即使配置了镜像仓库地址为http也不行，需要在拉取时加入--plain-http参数

坑3

runc在PATH中，但创建容器并运行时，还是会提示

exec: "runc": executable file not found in $PATH: unknown

原因比较蛋疼，containerd创建容器时，是根据config.conf中的PATH，也就是镜像默认的PATH路径去寻找runc的。如果你的PATH配置比较奇葩，那就有可能找不到。建议的做法是将/usr/bin、/usr/local/bin等常用目录全部建一个runc的软连接。