Support VCPU hotplug in QEMU guests

QEMU has a monitor command 'set_cpu' which allows a specific
CPU to be toggled between online& offline state. libvirt CPU
hotplug does not work in terms of individual indexes CPUs.
Thus to support this, we iteratively toggle the online state
when the total number of vCPUs is adjusted via libvirt

NB, currently untested since QEMU segvs when running this!

* src/qemu/qemu_driver.c: Toggle online state for CPUs when
  doing hotplug
* src/qemu/qemu_monitor.c, src/qemu/qemu_monitor.h,
  src/qemu/qemu_monitor_json.c, src/qemu/qemu_monitor_json.h,
  src/qemu/qemu_monitor_text.c, src/qemu/qemu_monitor_text.h: Add
  monitor API for toggling a CPU's online status via 'set_cpu

src/qemu/qemu_driver.c

@@ -4494,10 +4494,56 @@ cleanup:
 }
 
 
-static int qemudDomainSetVcpus(virDomainPtr dom,
+static int qemudDomainHotplugVcpus(virDomainObjPtr vm, unsigned int nvcpus)
-                               ATTRIBUTE_UNUSED unsigned int nvcpus) {
+{
+    qemuDomainObjPrivatePtr priv = vm->privateData;
+    int i, rc;
+    int ret = -1;
+
+    /* We need different branches here, because we want to offline
+     * in reverse order to onlining, so any partial fail leaves us in a
+     * reasonably sensible state */
+    if (nvcpus > vm->def->vcpus) {
+        for (i = vm->def->vcpus ; i < nvcpus ; i++) {
+            /* Online new CPU */
+            rc = qemuMonitorSetCPU(priv->mon, i, 1);
+            if (rc == 0)
+                goto unsupported;
+            if (rc < 0)
+                goto cleanup;
+
+            vm->def->vcpus++;
+        }
+    } else {
+        for (i = vm->def->vcpus - 1 ; i >= nvcpus ; i--) {
+            /* Offline old CPU */
+            rc = qemuMonitorSetCPU(priv->mon, i, 0);
+            if (rc == 0)
+                goto unsupported;
+            if (rc < 0)
+                goto cleanup;
+
+            vm->def->vcpus--;
+        }
+    }
+
+    ret = 0;
+
+cleanup:
+    return ret;
+
+unsupported:
+    qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                    _("cannot change vcpu count of this domain"));
+    goto cleanup;
+}
+
+
+static int qemudDomainSetVcpus(virDomainPtr dom, unsigned int nvcpus) {
     struct qemud_driver *driver = dom->conn->privateData;
     virDomainObjPtr vm;
+    const char * type;
+    int max;
     int ret = -1;
 
     qemuDriverLock(driver);
@@ -4518,8 +4564,31 @@ static int qemudDomainSetVcpus(virDomainPtr dom,
         goto cleanup;
     }
 
-    qemuReportError(VIR_ERR_NO_SUPPORT,
+    if (!(type = virDomainVirtTypeToString(vm->def->virtType))) {
-                     "%s", _("cpu hotplug not yet supported"));
+        qemuReportError(VIR_ERR_INTERNAL_ERROR,
+                        _("unknown virt type in domain definition '%d'"),
+                        vm->def->virtType);
+        goto endjob;
+    }
+
+    if ((max = qemudGetMaxVCPUs(NULL, type)) < 0) {
+        qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                        _("could not determine max vcpus for the domain"));
+        goto endjob;
+    }
+
+    if (nvcpus > max) {
+        qemuReportError(VIR_ERR_INVALID_ARG,
+                        _("requested vcpus is greater than max allowable"
+                          " vcpus for the domain: %d > %d"), nvcpus, max);
+        goto endjob;
+    }
+
+    ret = qemudDomainHotplugVcpus(vm, nvcpus);
+
+endjob:
+    if (qemuDomainObjEndJob(vm) == 0)
+        vm = NULL;
 
 cleanup:
     if (vm)

src/qemu/qemu_monitor.c

@@ -925,6 +925,20 @@ int qemuMonitorSetBalloon(qemuMonitorPtr mon,
     return ret;
 }
 
+
+int qemuMonitorSetCPU(qemuMonitorPtr mon, int cpu, int online)
+{
+    int ret;
+    DEBUG("mon=%p, fd=%d cpu=%d online=%d", mon, mon->fd, cpu, online);
+
+    if (mon->json)
+        ret = qemuMonitorJSONSetCPU(mon, cpu, online);
+    else
+        ret = qemuMonitorTextSetCPU(mon, cpu, online);
+    return ret;
+}
+
+
 int qemuMonitorEjectMedia(qemuMonitorPtr mon,
                           const char *devname)
 {

src/qemu/qemu_monitor.h

@@ -146,6 +146,8 @@ int qemuMonitorSetVNCPassword(qemuMonitorPtr mon,
                               const char *password);
 int qemuMonitorSetBalloon(qemuMonitorPtr mon,
                           unsigned long newmem);
+int qemuMonitorSetCPU(qemuMonitorPtr mon, int cpu, int online);
+
 
 /* XXX should we pass the virDomainDiskDefPtr instead
  * and hide devname details inside monitor. Reconsider

src/qemu/qemu_monitor_json.c

@@ -914,6 +914,50 @@ cleanup:
 }
 
 
+/*
+ * Returns: 0 if CPU hotplug not supported, +1 if CPU hotplug worked
+ * or -1 on failure
+ */
+int qemuMonitorJSONSetCPU(qemuMonitorPtr mon,
+                          int cpu, int online)
+{
+    int ret;
+    virJSONValuePtr cmd = qemuMonitorJSONMakeCommand("balloon",
+                                                     "U:cpu", (unsigned long long)cpu,
+                                                     "s:state", online ? "online" : "offline",
+                                                     NULL);
+    virJSONValuePtr reply = NULL;
+    if (!cmd)
+        return -1;
+
+    ret = qemuMonitorJSONCommand(mon, cmd, &reply);
+
+    if (ret == 0) {
+        /* XXX See if CPU soft-failed due to lack of ACPI */
+#if 0
+        if (qemuMonitorJSONHasError(reply, "DeviceNotActive") ||
+            qemuMonitorJSONHasError(reply, "KVMMissingCap"))
+            goto cleanup;
+#endif
+
+        /* See if any other fatal error occurred */
+        ret = qemuMonitorJSONCheckError(cmd, reply);
+
+        /* Real success */
+        if (ret == 0)
+            ret = 1;
+    }
+
+#if 0
+cleanup:
+#endif
+
+    virJSONValueFree(cmd);
+    virJSONValueFree(reply);
+    return ret;
+}
+
+
 int qemuMonitorJSONEjectMedia(qemuMonitorPtr mon,
                               const char *devname)
 {

src/qemu/qemu_monitor_json.h

@@ -59,6 +59,7 @@ int qemuMonitorJSONSetVNCPassword(qemuMonitorPtr mon,
                                   const char *password);
 int qemuMonitorJSONSetBalloon(qemuMonitorPtr mon,
                               unsigned long newmem);
+int qemuMonitorJSONSetCPU(qemuMonitorPtr mon, int cpu, int online);
 
 int qemuMonitorJSONEjectMedia(qemuMonitorPtr mon,
                               const char *devname);

src/qemu/qemu_monitor_text.c

@@ -797,6 +797,44 @@ int qemuMonitorTextSetBalloon(qemuMonitorPtr mon,
     return ret;
 }
 
+
+/*
+ * Returns: 0 if balloon not supported, +1 if balloon adjust worked
+ * or -1 on failure
+ */
+int qemuMonitorTextSetCPU(qemuMonitorPtr mon, int cpu, int online)
+{
+    char *cmd;
+    char *reply = NULL;
+    int ret = -1;
+
+    if (virAsprintf(&cmd, "set_cpu %d %s", cpu, online ? "online" : "offline") < 0) {
+        virReportOOMError();
+        return -1;
+    }
+
+    if (qemuMonitorCommand(mon, cmd, &reply) < 0) {
+        qemuReportError(VIR_ERR_OPERATION_FAILED,
+                        "%s", _("could nt change CPU online status"));
+        VIR_FREE(cmd);
+        return -1;
+    }
+    VIR_FREE(cmd);
+
+    /* If the command failed qemu prints: 'unknown command'
+     * No message is printed on success it seems */
+    if (strstr(reply, "\nunknown command:")) {
+        /* Don't set error - it is expected CPU onlining fails on many qemu - caller will handle */
+        ret = 0;
+    } else {
+        ret = 1;
+    }
+
+    VIR_FREE(reply);
+    return ret;
+}
+
+
 int qemuMonitorTextEjectMedia(qemuMonitorPtr mon,
                               const char *devname)
 {

src/qemu/qemu_monitor_text.h

@@ -61,6 +61,7 @@ int qemuMonitorTextSetVNCPassword(qemuMonitorPtr mon,
                                   const char *password);
 int qemuMonitorTextSetBalloon(qemuMonitorPtr mon,
                               unsigned long newmem);
+int qemuMonitorTextSetCPU(qemuMonitorPtr mon, int cpu, int online);
 
 int qemuMonitorTextEjectMedia(qemuMonitorPtr mon,
                               const char *devname);