vsock 开发实践-virtio-vsock

vsock 开发实践-virtio-vsock

一、背景

我们知道在不使用tcp/ip通信的情况下，传统实现guest和host之间的通信主要是基于virtio-serial 串口通信，但是virtio-serial 有一定的局限性。

特性	Virtio-serial	Virtio-vsock
底层机制	基于虚拟串口（字符设备），模拟传统物理串口	基于套接字（Socket），共享内存直接通信
数据传输模式	流式传输（类似字节流）	面向消息的数据包传输
寻址方式	通过设备路径（如 /dev/vport0p1）	通过 CID（Context ID）和端口号寻址
典型延迟	较高（依赖虚拟中断机制）	低至微秒级（绕过网络栈）

适用场景对比

场景	推荐技术	理由
虚拟机控制台交互	Virtio-serial	兼容性强，适合低频次命令行操作
实时日志采集	Virtio-vsock	高吞吐、低延迟，避免日志堆积
热迁移元数据同步	Virtio-vsock	减少网络抖动对迁移过程的影响
容器与 Host 通信	Virtio-vsock	适配 Kata Containers 等轻量级沙箱架构
传统设备模拟	Virtio-serial	模拟串口/打印机等设备无需协议改造

总的来说，virtio-serial 对内核无要求，兼容性更强且稳定，但是性能低，适合于小规模数据传输的通用场景下使用。而virtio-vsock 对内核有要求，更适合于高吞吐低延迟对性能要求更高的场景下。

二、安装vsock

使用 virtio-vsock 对hypervisor以及guest 有一定的限制要求，两者的kernel 都必须 4.8+ ，同时qemu 2.8+。

• Host kernel requirements: CONFIG_VHOST_VSOCK=m
• Guest kernel requirements: CONFIG_VIRTIO_VSOCKETS=m

2.1 宿主机

加载vhost_vsock内核模块

hypervisor # cat /boot/config-`uname -r` | grep -i VHOST_VSOCK
hypervisor #modprobe vhost_vsock
hypervisor #ls -l /dev/vhost-vsock
crw------- 1 root root 10, 53 May  4 11:55 /dev/vhost-vsock
hypervisor # ls -l /dev/vsock
crw-rw-rw- 1 root root 10, 54 May  4 11:55 /dev/vsock

2.2 qemu启动

当前主流虚拟化平台（如 QEMU 2.8+ 和 Libvirt 4.4+）已原生支持 vhost-vsock-pci 设备，适用于云原生和高性能通信场景。

libvirt 配置vsock device

A vsock host/guest interface. The model attribute defaults to virtio. Since 5.2.0 model can also be ‘virtio-transitional’ or ‘virtio-non-transitional’, see virtio device models for more details. The optional attribute address of the cid element specifies the CID assigned to the guest. If the attribute auto is set to yes, libvirt will assign a free CID automatically on domain startup. Since 4.4.0 The optional driver element allows to specify virtio options, see Virtio-related options for more details. Since 7.1.0

<devices>
  <vsock model="virtio">
    <!-- 自动分配 Guest CID -->  
    <cid auto="yes"/>
    <!-- 可选：绑定固定 CID（需全局唯一） -->  
    <!-- <cid address="3"/> -->
  </vsock>
</devices>

qemu 启动加上 vhost-vsock-pci 参数， 并指定虚机唯一的cid编号

hypervisor #sudo qemu-system-x86_64 -m 4G -hda /home/matt/ubuntuvm0.img \
             -device vhost-vsock-pci,id=vhost-vsock-pci0,guest-cid=3 -vnc :0 \
             --enable-kvm

2.3 虚机侧

qemu 启动的虚机理应已经有vsock device，检查一下 vsock

(上文提到需要内核4.8+, 故CentOS8下能看到而CentOS7 看不到)

vm # cat /boot/config-`uname -r` | grep -i VIRTIO_VSOCK
vm # ls -l /dev/vsock
crw-rw-rw- 1 root root 10, 55 May  4 13:21 /dev/vsock

三、Host/Guest 通信开发

启动一台 CentOS 8.2 x86_64 的虚拟机用于开发测试

3.1 hypervisor/server端

python语言

#!/usr/bin/env python3

import socket

CID = socket.VMADDR_CID_HOST
PORT = 9999

s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
s.bind((CID, PORT))
s.listen()
(conn, (remote_cid, remote_port)) = s.accept()

print(f"Connection opened by cid={remote_cid} port={remote_port}")

while True:
    buf = conn.recv(64)
    if not buf:
        break

    print(f"Received bytes: {buf}")

c 语言

#include <sys/socket.h>
#include <linux/vm_sockets.h>
#include <string.h>
#include <stdio.h>

int main()
{
	int s = socket(AF_VSOCK, SOCK_STREAM, 0);

	struct sockaddr_vm addr;
	memset(&addr, 0, sizeof(struct sockaddr_vm));
	addr.svm_family = AF_VSOCK;
	addr.svm_port = 9999;
	addr.svm_cid = VMADDR_CID_HOST;

	bind(s, &addr, sizeof(struct sockaddr_vm));

	listen(s, 0);

	struct sockaddr_vm peer_addr;
	socklen_t peer_addr_size = sizeof(struct sockaddr_vm);
	int peer_fd = accept(s, &peer_addr, &peer_addr_size);

	char buf[64];
	size_t msg_len;
	while ((msg_len = recv(peer_fd, &buf, 64, 0)) > 0) {
		printf("Received %lu bytes: %.*s\n", msg_len, msg_len, buf);
	}

	return 0;
}

go 语言

// Retrieve host's context ID from /dev/vsock. More on this later.
cid := localContextID()

// Establish a connection-oriented VM socket.
socket, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM, 0)
if err != nil {
    return err
}

// Bind socket to local context ID, port 1024.
sockaddr := &unix.SockaddrVM{
    CID:  cid,
    Port: 1024,
}
if err := unix.Bind(socket, sockaddr); err != nil {
    return err
}

// Listen for up to 32 incoming connections.
fd, err := unix.Listen(socket, 32)
if err != nil {
    return err
}

// Use fd to read and write data to and from a VM.

3.2 guest/client 端

python语言

#!/usr/bin/env python3

import socket

CID = socket.VMADDR_CID_HOST
PORT = 9999

s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
s.connect((CID, PORT))
s.sendall(b"Hello, world!")
s.close()

c 语言

#include <sys/socket.h>
#include <linux/vm_sockets.h>
#include <string.h>

int main()
{
	int s = socket(AF_VSOCK, SOCK_STREAM, 0);

	struct sockaddr_vm addr;
	memset(&addr, 0, sizeof(struct sockaddr_vm));
	addr.svm_family = AF_VSOCK;
	addr.svm_port = 9999;
	addr.svm_cid = VMADDR_CID_HOST;

	connect(s, &addr, sizeof(struct sockaddr_vm));

	send(s, "Hello, world!", 13, 0);

	close(s);

	return 0;
}

go 语言

// Establish a connection-oriented VM socket.
socket, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM, 0)
if err != nil {
    return err
}

// Connect socket to hypervisor context ID, port 1024.
sockaddr := &unix.SockaddrVM{
    CID:  2,
    Port: 1024,
}
if err := unix.Connect(socket, sockaddr); err != nil {
   return err
}

// Use fd to read and write data to and from the hypervisor.

3.3 wireshark抓包

ip link add type vsockmon
ip link set vsockmon0 up
wireshark -k -i vsockmon0
ip link set vsockmon0 down
ip link del vsockmon0

参考：

https://mdlayher.com/blog/linux-vm-sockets-in-go/

https://wiki.qemu.org/Features/VirtioVsock

https://libvirt.org/formatdomain.html

https://gist.github.com/nrdmn/7971be650919b112343b1cb2757a3fe6#c

https://stefano-garzarella.github.io/posts/2020-02-20-vsock-nested-vms-loopback/#local-communication

https://libvirt.org/formatdomain.html#vsock