vhost-user: support registering external host notifiers (44866521) · Commits · SUMMER2020 / students / proj-2021291

docs/interop/vhost-user.txt

+33 −0

Original line number	Diff line number	Diff line
		@@ -132,6 +132,16 @@ Depending on the request type, payload can be:
		Payload: Size bytes array holding the contents of the virtio
		device's configuration space

		* Vring area description
		-----------------------
		\| u64 \| size \| offset \|
		-----------------------

		u64: a 64-bit integer contains vring index and flags
		Size: a 64-bit size of this area
		Offset: a 64-bit offset of this area from the start of the
		supplied file descriptor

		In QEMU the vhost-user message is implemented with the following struct:

		typedef struct VhostUserMsg {
		@@ -146,6 +156,7 @@ typedef struct VhostUserMsg {
		VhostUserLog log;
		struct vhost_iotlb_msg iotlb;
		VhostUserConfig config;
		VhostUserVringArea area;
		};
		} QEMU_PACKED VhostUserMsg;

		@@ -385,6 +396,7 @@ Protocol features
		#define VHOST_USER_PROTOCOL_F_PAGEFAULT 8
		#define VHOST_USER_PROTOCOL_F_CONFIG 9
		#define VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD 10
		#define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 11

		Master message types
		--------------------
		@@ -782,6 +794,27 @@ Slave message types
		the VHOST_USER_NEED_REPLY flag, master must respond with zero when
		operation is successfully completed, or non-zero otherwise.

		* VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG

		Id: 3
		Equivalent ioctl: N/A
		Slave payload: vring area description
		Master payload: N/A

		Sets host notifier for a specified queue. The queue index is contained
		in the u64 field of the vring area description. The host notifier is
		described by the file descriptor (typically it's a VFIO device fd) which
		is passed as ancillary data and the size (which is mmap size and should
		be the same as host page size) and offset (which is mmap offset) carried
		in the vring area description. QEMU can mmap the file descriptor based
		on the size and offset to get a memory range. Registering a host notifier
		means mapping this memory range to the VM as the specified queue's notify
		MMIO region. Slave sends this request to tell QEMU to de-register the
		existing notifier if any and register the new notifier if the request is
		sent with a file descriptor.
		This request should be sent only when VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
		protocol feature has been successfully negotiated.

		VHOST_USER_PROTOCOL_F_REPLY_ACK:
		-------------------------------
		The original vhost-user specification only demands replies for certain

hw/virtio/vhost-user.c

+113 −0

Original line number	Diff line number	Diff line
		@@ -13,6 +13,7 @@
		#include "hw/virtio/vhost.h"
		#include "hw/virtio/vhost-user.h"
		#include "hw/virtio/vhost-backend.h"
		#include "hw/virtio/virtio.h"
		#include "hw/virtio/virtio-net.h"
		#include "chardev/char-fe.h"
		#include "sysemu/kvm.h"
		@@ -50,6 +51,7 @@ enum VhostUserProtocolFeature {
		VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
		VHOST_USER_PROTOCOL_F_CONFIG = 9,
		VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
		VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
		VHOST_USER_PROTOCOL_F_MAX
		};

		@@ -94,6 +96,7 @@ typedef enum VhostUserSlaveRequest {
		VHOST_USER_SLAVE_NONE = 0,
		VHOST_USER_SLAVE_IOTLB_MSG = 1,
		VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
		VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
		VHOST_USER_SLAVE_MAX
		} VhostUserSlaveRequest;

		@@ -138,6 +141,12 @@ static VhostUserConfig c __attribute__ ((unused));
		+ sizeof(c.size) \
		+ sizeof(c.flags))

		typedef struct VhostUserVringArea {
		uint64_t u64;
		uint64_t size;
		uint64_t offset;
		} VhostUserVringArea;

		typedef struct {
		VhostUserRequest request;

		@@ -159,6 +168,7 @@ typedef union {
		struct vhost_iotlb_msg iotlb;
		VhostUserConfig config;
		VhostUserCryptoSession session;
		VhostUserVringArea area;
		} VhostUserPayload;

		typedef struct VhostUserMsg {
		@@ -640,9 +650,37 @@ static int vhost_user_set_vring_num(struct vhost_dev *dev,
		return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
		}

		static void vhost_user_host_notifier_restore(struct vhost_dev *dev,
		int queue_idx)
		{
		struct vhost_user *u = dev->opaque;
		VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
		VirtIODevice *vdev = dev->vdev;

		if (n->addr && !n->set) {
		virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true);
		n->set = true;
		}
		}

		static void vhost_user_host_notifier_remove(struct vhost_dev *dev,
		int queue_idx)
		{
		struct vhost_user *u = dev->opaque;
		VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
		VirtIODevice *vdev = dev->vdev;

		if (n->addr && n->set) {
		virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
		n->set = false;
		}
		}

		static int vhost_user_set_vring_base(struct vhost_dev *dev,
		struct vhost_vring_state *ring)
		{
		vhost_user_host_notifier_restore(dev, ring->index);

		return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
		}

		@@ -676,6 +714,8 @@ static int vhost_user_get_vring_base(struct vhost_dev *dev,
		.hdr.size = sizeof(msg.payload.state),
		};

		vhost_user_host_notifier_remove(dev, ring->index);

		if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
		return -1;
		}
		@@ -849,6 +889,66 @@ static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
		return ret;
		}

		static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
		VhostUserVringArea *area,
		int fd)
		{
		int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
		size_t page_size = qemu_real_host_page_size;
		struct vhost_user *u = dev->opaque;
		VhostUserState *user = u->user;
		VirtIODevice *vdev = dev->vdev;
		VhostUserHostNotifier *n;
		void *addr;
		char *name;

		if (!virtio_has_feature(dev->protocol_features,
		VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) \|\|
		vdev == NULL \|\| queue_idx >= virtio_get_num_queues(vdev)) {
		return -1;
		}

		n = &user->notifier[queue_idx];

		if (n->addr) {
		virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
		object_unparent(OBJECT(&n->mr));
		munmap(n->addr, page_size);
		n->addr = NULL;
		}

		if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
		return 0;
		}

		/* Sanity check. */
		if (area->size != page_size) {
		return -1;
		}

		addr = mmap(NULL, page_size, PROT_READ \| PROT_WRITE, MAP_SHARED,
		fd, area->offset);
		if (addr == MAP_FAILED) {
		return -1;
		}

		name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
		user, queue_idx);
		memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
		page_size, addr);
		g_free(name);

		if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
		munmap(addr, page_size);
		return -1;
		}

		n->addr = addr;
		n->set = true;

		return 0;
		}

		static void slave_read(void *opaque)
		{
		struct vhost_dev *dev = opaque;
		@@ -917,6 +1017,10 @@ static void slave_read(void *opaque)
		case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
		ret = vhost_user_slave_handle_config_change(dev);
		break;
		case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
		ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
		fd[0]);
		break;
		default:
		error_report("Received unexpected msg type.");
		ret = -EINVAL;
		@@ -1648,6 +1752,15 @@ VhostUserState *vhost_user_init(void)

		void vhost_user_cleanup(VhostUserState *user)
		{
		int i;

		for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
		if (user->notifier[i].addr) {
		object_unparent(OBJECT(&user->notifier[i].mr));
		munmap(user->notifier[i].addr, qemu_real_host_page_size);
		user->notifier[i].addr = NULL;
		}
		}
		}

		const VhostOps user_ops = {

include/hw/virtio/vhost-user.h

+8 −0

Original line number	Diff line number	Diff line
		@@ -9,9 +9,17 @@
		#define HW_VIRTIO_VHOST_USER_H

		#include "chardev/char-fe.h"
		#include "hw/virtio/virtio.h"

		typedef struct VhostUserHostNotifier {
		MemoryRegion mr;
		void *addr;
		bool set;
		} VhostUserHostNotifier;

		typedef struct VhostUserState {
		CharBackend *chr;
		VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX];
		} VhostUserState;

		VhostUserState *vhost_user_init(void);