!5 virtio: add a new vcpu watchdog

Merge pull request !5 from Sebastian2020/openkylin/yangtze

hw/virtio/Kconfig

@@ -80,3 +80,8 @@ config VHOST_USER_FS
     bool
     default y
     depends on VIRTIO && VHOST_USER
+
+config VIRTIO_VCPU_WATCHDOG
+    bool
+    default y
+    depends on VIRTIO

hw/virtio/meson.build

@@ -29,6 +29,7 @@ virtio_ss.add(when: 'CONFIG_VIRTIO_IOMMU', if_true: files('virtio-iommu.c'))
 virtio_ss.add(when: 'CONFIG_VIRTIO_MEM', if_true: files('virtio-mem.c'))
 virtio_ss.add(when: 'CONFIG_VHOST_USER_I2C', if_true: files('vhost-user-i2c.c'))
 virtio_ss.add(when: 'CONFIG_VHOST_USER_RNG', if_true: files('vhost-user-rng.c'))
+virtio_ss.add(when: 'CONFIG_VIRTIO_VCPU_WATCHDOG', if_true: files('virtio-vcpu-watchdog.c'))
 
 virtio_pci_ss = ss.source_set()
 virtio_pci_ss.add(when: 'CONFIG_VHOST_VSOCK', if_true: files('vhost-vsock-pci.c'))
@@ -54,6 +55,7 @@ virtio_pci_ss.add(when: 'CONFIG_VIRTIO_SERIAL', if_true: files('virtio-serial-pc
 virtio_pci_ss.add(when: 'CONFIG_VIRTIO_PMEM', if_true: files('virtio-pmem-pci.c'))
 virtio_pci_ss.add(when: 'CONFIG_VIRTIO_IOMMU', if_true: files('virtio-iommu-pci.c'))
 virtio_pci_ss.add(when: 'CONFIG_VIRTIO_MEM', if_true: files('virtio-mem-pci.c'))
+virtio_pci_ss.add(when: 'CONFIG_VIRTIO_VCPU_WATCHDOG', if_true: files('virtio-vcpu-watchdog-pci.c'))
 
 virtio_ss.add_all(when: 'CONFIG_VIRTIO_PCI', if_true: virtio_pci_ss)
 

hw/virtio/trace-events

@@ -140,3 +140,9 @@ virtio_mem_state_response(uint16_t state) "state=%" PRIu16
 virtio_pmem_flush_request(void) "flush request"
 virtio_pmem_response(void) "flush response"
 virtio_pmem_flush_done(int type) "fsync return=%d"
+
+# virtio-vcpu-watchdog.c
+virtio_receive_vcpu_info(size_t len, int cpu, uint32_t is_initialized, uint32_t ticks) "len=%zd cpu=%d is_initialized=%d ticks=%d"
+virtio_vcpu_check_ticks(int cpu_id, uint32_t ticks) "cpu_id=%d ticks=%d"
+virtio_vcpu_check_reset(int cpu_id) "CPU:%d is stall need to reset vm"
+virtio_vcpu_watchdog_process(int thread_id) "vcpu thread id:%d"

hw/virtio/virtio-vcpu-watchdog-pci.c

@@ -0,0 +1,86 @@
+/*
+ * Virtio cpu watchdog PCI Bindings
+ *
+ * Copyright 2023 Kylin, Inc.
+ * Copyright 2023 Hao Zhang <zhanghao1@kylinos.cn>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+
+#include "hw/virtio/virtio-pci.h"
+#include "hw/virtio/virtio-vcpu-watchdog.h"
+#include "qapi/error.h"
+#include "qemu/module.h"
+
+typedef struct VirtIOCpuWatchdogPCI VirtIOCpuWatchdogPCI;
+
+/*
+ * virtio-cpu-watchdog-pci: This extends VirtioPCIProxy.
+ */
+#define TYPE_VIRTIO_CPU_WATCHDOG_PCI "virtio-vcpu-watchdog-pci"
+DECLARE_INSTANCE_CHECKER(VirtIOCpuWatchdogPCI, VIRTIO_CPU_WATCHDOG_PCI,
+                         TYPE_VIRTIO_CPU_WATCHDOG_PCI)
+
+struct VirtIOCpuWatchdogPCI {
+    VirtIOPCIProxy parent_obj;
+    VirtIOCPUWATCHDOG vdev;
+};
+
+static Property vcpu_watchdog_properties[] = {
+    DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors,
+                       DEV_NVECTORS_UNSPECIFIED),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_vcpu_watchdog_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
+{
+    VirtIOCpuWatchdogPCI *dev = VIRTIO_CPU_WATCHDOG_PCI(vpci_dev);
+    DeviceState *vdev = DEVICE(&dev->vdev);
+
+    if (vpci_dev->nvectors == DEV_NVECTORS_UNSPECIFIED) {
+        vpci_dev->nvectors = 1;
+    }
+
+    if (!qdev_realize(vdev, BUS(&vpci_dev->bus), errp)) {
+        return;
+    }
+}
+
+static void virtio_vcpu_watchdog_pci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
+    PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
+
+    k->realize = virtio_vcpu_watchdog_pci_realize;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    device_class_set_props(dc, vcpu_watchdog_properties);
+    pcidev_k->revision = VIRTIO_PCI_ABI_VERSION;
+    pcidev_k->class_id = PCI_CLASS_OTHERS;
+}
+
+static void virtio_vcpu_watchdog_init(Object *obj)
+{
+    VirtIOCpuWatchdogPCI *dev = VIRTIO_CPU_WATCHDOG_PCI(obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_VCPU_WATCHDOG);
+}
+
+static const VirtioPCIDeviceTypeInfo virtio_vcpu_watchdog_pci_info = {
+    .generic_name  = TYPE_VIRTIO_CPU_WATCHDOG_PCI,
+    .instance_size = sizeof(VirtIOCpuWatchdogPCI),
+    .instance_init = virtio_vcpu_watchdog_init,
+    .class_init    = virtio_vcpu_watchdog_pci_class_init,
+};
+
+static void virtio_vcpu_watchdog_pci_register(void)
+{
+    virtio_pci_types_register(&virtio_vcpu_watchdog_pci_info);
+}
+
+type_init(virtio_vcpu_watchdog_pci_register)

hw/virtio/virtio-vcpu-watchdog.c

@@ -0,0 +1,226 @@
+/*
+ * A virtio device implementing a vcpu watchdog.
+ *
+ * Copyright 2023 Kylin, Inc.
+ * Copyright 2023 zhanghao1 <zhanghao1@kylinos.cn>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/iov.h"
+#include "qemu/module.h"
+#include "qemu/bswap.h"
+#include "hw/virtio/virtio.h"
+#include "hw/virtio/virtio-vcpu-watchdog.h"
+#include "qom/object_interfaces.h"
+#include "trace.h"
+#include "standard-headers/linux/virtio_ids.h"
+#include "hw/virtio/virtio-access.h"
+#include "hw/boards.h"
+#include "sysemu/cpus.h"
+#include "sysemu/runstate.h"
+#include "sysemu/watchdog.h"
+
+#define VCPU_DEFAULT_CLOCK_HZ (5)
+#define VCPU_DEFAULT_TIMEOUT_SEC (8)
+#define MSEC_PER_SEC 1000
+#define PROCSTAT_UTIME_INDX 13
+#define PROCSTAT_GUEST_TIME_INDX 42
+
+struct vcpu_info {
+    uint32_t cpu_id;
+    uint32_t is_initialized;
+    uint32_t ticks;
+    uint64_t not_running_last_timestamp;
+};
+
+static VirtIOCPUWATCHDOG *vwdt;
+
+static bool is_guest_ready(VirtIOCPUWATCHDOG *vwdt)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(vwdt);
+    if (virtio_queue_ready(vwdt->vq) &&
+        (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK)) {
+        return true;
+    }
+    return false;
+}
+
+/* receive data from guest */
+static void receive_vcpu_info(void *opaque, void *buf, size_t size)
+{
+    VirtIOCPUWATCHDOG *vwdt = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(vwdt);
+    g_autofree VirtQueueElement *elem;
+    size_t len;
+
+    if (!is_guest_ready(vwdt)) {
+        return;
+    }
+
+    elem = virtqueue_pop(vwdt->vq, sizeof(VirtQueueElement));
+    if (!elem) {
+        return;
+    }
+
+    len = iov_size(elem->out_sg, elem->out_num);
+
+    len = iov_to_buf(elem->out_sg, elem->out_num,
+                     0, buf, len);
+
+    int cpu = virtio_ldl_p(vdev, &((struct vcpu_info *)buf)->cpu_id);
+    trace_virtio_receive_vcpu_info(len, cpu,
+                                   ((struct vcpu_info *)buf)->is_initialized,
+                                   ((struct vcpu_info *)buf)->ticks);
+
+    virtqueue_push(vwdt->vq, elem, len);
+    virtio_notify(vdev, vwdt->vq);
+}
+
+static void vcpu_check(void *opaque)
+{
+    int *cpu_id = (int *)opaque;
+
+    struct vcpu_info *priv = vwdt->recv_buf[*cpu_id];
+
+    trace_virtio_vcpu_check_ticks(*cpu_id, priv->ticks);
+
+    priv->ticks -= 1;
+
+    if (priv->ticks <= 0) {
+        /* cpu is stall, reset vm */
+        trace_virtio_vcpu_check_reset(*cpu_id);
+        watchdog_perform_action();
+    }
+
+    int64_t expire_timer = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
+    expire_timer += (MSEC_PER_SEC / VCPU_DEFAULT_CLOCK_HZ);
+    timer_mod(vwdt->timer[*cpu_id], expire_timer);
+}
+
+static void virtio_vcpu_watchdog_process(VirtIOCPUWATCHDOG *vwdt)
+{
+    int i = 0;
+    struct vcpu_info recv_buf;
+
+    if (!is_guest_ready(vwdt)) {
+        return;
+    }
+
+    receive_vcpu_info(vwdt, &recv_buf, sizeof(recv_buf));
+
+    for (i = 0; i < vwdt->num_timers; i++) {
+        if (vwdt->recv_buf[i]) {
+            if (vwdt->recv_buf[i]->cpu_id == recv_buf.cpu_id) {
+                /* update ticks */
+                vwdt->recv_buf[i]->is_initialized = 1;
+                vwdt->recv_buf[i]->ticks = recv_buf.ticks;
+            }
+        } else {
+            break;
+        }
+    }
+
+    if (i != vwdt->num_timers) {
+        struct vcpu_info *priv = g_new0(struct vcpu_info, 1);
+
+        memcpy(priv, &recv_buf, sizeof(struct vcpu_info));
+        vwdt->recv_buf[i] = priv;
+        vwdt->timer[i] = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+                                            vcpu_check, &priv->cpu_id);
+
+        int64_t expire_timer = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
+        expire_timer += (MSEC_PER_SEC / VCPU_DEFAULT_CLOCK_HZ);
+        timer_mod(vwdt->timer[i], expire_timer);
+
+        CPUState *cpu = qemu_get_cpu(recv_buf.cpu_id);
+        if (!cpu) {
+            return;
+        }
+        trace_virtio_vcpu_watchdog_process(cpu->thread_id);
+    }
+}
+
+static void handle_input(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOCPUWATCHDOG *vwdt = VIRTIO_VCPU_WATCHDOG(vdev);
+    virtio_vcpu_watchdog_process(vwdt);
+}
+
+static uint64_t get_features(VirtIODevice *vdev, uint64_t f, Error **errp)
+{
+    return f;
+}
+
+static void virtio_vcpu_watchdog_device_realize(DeviceState *dev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    vwdt = VIRTIO_VCPU_WATCHDOG(dev);
+
+    virtio_init(vdev, VIRTIO_ID_WATCHDOG, 0);
+
+    vwdt->vq = virtio_add_queue(vdev, 1024, handle_input);
+
+    MachineState *ms = MACHINE(qdev_get_machine());
+    unsigned int smp_cpus = ms->smp.cpus;
+
+    vwdt->timer = g_new0(struct QEMUTimer *, smp_cpus);
+
+    vwdt->recv_buf = g_new0(struct vcpu_info *, smp_cpus);
+
+    vwdt->num_timers = smp_cpus;
+}
+
+static void virtio_vcpu_watchdog_device_unrealize(DeviceState *dev)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VirtIOCPUWATCHDOG  *vwdt = VIRTIO_VCPU_WATCHDOG(dev);
+
+    g_free(vwdt->timer);
+    g_free(vwdt->recv_buf);
+    virtio_cleanup(vdev);
+}
+
+static const VMStateDescription vmstate_virtio_vcpu_watchdog = {
+    .name = "virtio-vcpu-watchdog",
+    .minimum_version_id = 1,
+    .version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_VIRTIO_DEVICE,
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static Property virtio_vcpu_watchdog_properties[] = {
+};
+
+static void virtio_vcpu_watchdog_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
+
+    device_class_set_props(dc, virtio_vcpu_watchdog_properties);
+    dc->vmsd = &vmstate_virtio_vcpu_watchdog;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    vdc->realize = virtio_vcpu_watchdog_device_realize;
+    vdc->unrealize = virtio_vcpu_watchdog_device_unrealize;
+    vdc->get_features = get_features;
+}
+
+static const TypeInfo virtio_vcpu_watchdog_info = {
+    .name = TYPE_VIRTIO_VCPU_WATCHDOG,
+    .parent = TYPE_VIRTIO_DEVICE,
+    .instance_size = sizeof(VirtIOCPUWATCHDOG),
+    .class_init = virtio_vcpu_watchdog_class_init,
+};
+
+static void virtio_register_types(void)
+{
+    type_register_static(&virtio_vcpu_watchdog_info);
+}
+
+type_init(virtio_register_types)

include/hw/virtio/virtio-vcpu-watchdog.h

@@ -0,0 +1,37 @@
+/*
+ * Virtio cpu watchdog Support
+ *
+ * Copyright Kylin, Inc. 2023
+ * Copyright zhanghao1 <zhanghao1@kylinos.cn>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#ifndef QEMU_VIRTIO_CPU_WATCHDOG_H
+#define QEMU_VIRTIO_CPU_WATCHDOG_H
+
+#include "hw/virtio/virtio.h"
+
+#define TYPE_VIRTIO_VCPU_WATCHDOG "virtio-vcpu-watchdog-device"
+
+#define VIRTIO_VCPU_WATCHDOG(obj) \
+        OBJECT_CHECK(VirtIOCPUWATCHDOG, (obj), TYPE_VIRTIO_VCPU_WATCHDOG)
+#define VIRTIO_CPU_WATCHDOG_GET_PARENT_CLASS(obj) \
+        OBJECT_GET_PARENT_CLASS(obj, TYPE_VIRTIO_VCPU_WATCHDOG)
+typedef struct VirtIOCPUWATCHDOG {
+    VirtIODevice parent_obj;
+
+    /* Only one vq - guest puts message on it when vcpu is stall */
+    VirtQueue *vq;
+
+    QEMUTimer **timer;
+    int num_timers;
+
+    struct vcpu_info **recv_buf;
+
+    uint64_t not_running_last_timestamp;
+} VirtIOCPUWATCHDOG;
+
+#endif