openkylin
/
libvirt
mirror of https://gitee.com/openkylin/libvirt.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

qemu: command: Add support for sparse vcpu topologies 

Add support for using the new approach to hotplug vcpus using device_add
during startup of qemu to allow sparse vcpu topologies.

There are a few limitations imposed by qemu on the supported
configuration:
- vcpu0 needs to be always present and not hotpluggable
- non-hotpluggable cpus need to be ordered at the beginning
- order of the vcpus needs to be unique for every single hotpluggable
  entity

Qemu also doesn't really allow to query the information necessary to
start a VM with the vcpus directly on the commandline. Fortunately they
can be hotplugged during startup.

The new hotplug code uses the following approach:
- non-hotpluggable vcpus are counted and put to the -smp option
- qemu is started
- qemu is queried for the necessary information
- the configuration is checked
- the hotpluggable vcpus are hotplugged
- vcpus are started

This patch adds a lot of checking code and enables the support to
specify the individual vcpu element with qemu.
This commit is contained in:
Peter Krempa 2016-08-04 14:36:24 +02:00
parent 20ef1232ec
commit 9eb9106ea5
7 changed files with 325 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/formatdomain.html.in
View File

@ -580,6 +580,11 @@
Note that providing state for individual cpus may be necessary to enable
support of addressable vCPU hotplug and this feature may not be
supported by all hypervisors.
For QEMU the following conditions are required. Vcpu 0 needs to be
enabled and non-hotpluggable. On PPC64 along with it vcpus that are in
the same core need to be enabled as well. All non-hotpluggable cpus
present at boot need to be grouped after vcpu 0.
<span class="since">Since 2.2.0 (QEMU only)</span>
</dd>
</dl>

20
src/qemu/qemu_command.c
View File

@ -7082,17 +7082,29 @@ qemuBuildMachineCommandLine(virCommandPtr cmd,
static int
qemuBuildSmpCommandLine(virCommandPtr cmd,
const virDomainDef *def)
virDomainDefPtr def)
{
char *smp;
virBuffer buf = VIR_BUFFER_INITIALIZER;
unsigned int maxvcpus = virDomainDefGetVcpusMax(def);
unsigned int nvcpus = 0;
virDomainVcpuDefPtr vcpu;
size_t i;
/* count non-hotpluggable enabled vcpus. Hotpluggable ones will be added
* in a different way */
for (i = 0; i < maxvcpus; i++) {
vcpu = virDomainDefGetVcpu(def, i);
if (vcpu->online && vcpu->hotpluggable == VIR_TRISTATE_BOOL_NO)
nvcpus++;
}
virCommandAddArg(cmd, "-smp");
virBufferAsprintf(&buf, "%u", virDomainDefGetVcpus(def));
virBufferAsprintf(&buf, "%u", nvcpus);
if (virDomainDefHasVcpusOffline(def))
virBufferAsprintf(&buf, ",maxcpus=%u", virDomainDefGetVcpusMax(def));
if (nvcpus != maxvcpus)
virBufferAsprintf(&buf, ",maxcpus=%u", maxvcpus);
/* sockets, cores, and threads are either all zero
* or all non-zero, thus checking one of them is enough */
if (def->cpu && def->cpu->sockets) {

76
src/qemu/qemu_domain.c
View File

@ -2252,6 +2252,76 @@ qemuDomainRecheckInternalPaths(virDomainDefPtr def,
}
static int
qemuDomainDefVcpusPostParse(virDomainDefPtr def)
{
unsigned int maxvcpus = virDomainDefGetVcpusMax(def);
virDomainVcpuDefPtr vcpu;
virDomainVcpuDefPtr prevvcpu;
size_t i;
bool has_order = false;
/* vcpu 0 needs to be present, first, and non-hotpluggable */
vcpu = virDomainDefGetVcpu(def, 0);
if (!vcpu->online) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("vcpu 0 can't be offline"));
return -1;
}
if (vcpu->hotpluggable == VIR_TRISTATE_BOOL_YES) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("vcpu0 can't be hotpluggable"));
return -1;
}
if (vcpu->order != 0 && vcpu->order != 1) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("vcpu0 must be enabled first"));
return -1;
}
if (vcpu->order != 0)
has_order = true;
prevvcpu = vcpu;
/* all online vcpus or non online vcpu need to have order set */
for (i = 1; i < maxvcpus; i++) {
vcpu = virDomainDefGetVcpu(def, i);
if (vcpu->online &&
(vcpu->order != 0) != has_order) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("all vcpus must have either set or unset order"));
return -1;
}
/* few conditions for non-hotpluggable (thus online) vcpus */
if (vcpu->hotpluggable == VIR_TRISTATE_BOOL_NO) {
/* they can be ordered only at the beginning */
if (prevvcpu->hotpluggable == VIR_TRISTATE_BOOL_YES) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("online non-hotpluggable vcpus need to be "
"ordered prior to hotplugable vcpus"));
return -1;
}
/* they need to be in order (qemu doesn't support any order yet).
* Also note that multiple vcpus may share order on some platforms */
if (prevvcpu->order > vcpu->order) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("online non-hotpluggable vcpus must be ordered "
"in ascending order"));
return -1;
}
}
prevvcpu = vcpu;
}
return 0;
}
static int
qemuDomainDefPostParse(virDomainDefPtr def,
virCapsPtr caps,
@ -2307,6 +2377,9 @@ qemuDomainDefPostParse(virDomainDefPtr def,
if (virSecurityManagerVerify(driver->securityManager, def) < 0)
goto cleanup;
if (qemuDomainDefVcpusPostParse(def) < 0)
goto cleanup;
ret = 0;
cleanup:
virObjectUnref(qemuCaps);
@ -2709,7 +2782,8 @@ virDomainDefParserConfig virQEMUDriverDomainDefParserConfig = {
.deviceValidateCallback = qemuDomainDeviceDefValidate,
.features = VIR_DOMAIN_DEF_FEATURE_MEMORY_HOTPLUG |
VIR_DOMAIN_DEF_FEATURE_OFFLINE_VCPUPIN
VIR_DOMAIN_DEF_FEATURE_OFFLINE_VCPUPIN |
VIR_DOMAIN_DEF_FEATURE_INDIVIDUAL_VCPUS,
};

178
src/qemu/qemu_process.c
View File

@ -4760,6 +4760,172 @@ qemuProcessSetupIOThreads(virDomainObjPtr vm)
}
static int
qemuProcessValidateHotpluggableVcpus(virDomainDefPtr def)
{
virDomainVcpuDefPtr vcpu;
virDomainVcpuDefPtr subvcpu;
qemuDomainVcpuPrivatePtr vcpupriv;
unsigned int maxvcpus = virDomainDefGetVcpusMax(def);
size_t i = 0;
size_t j;
virBitmapPtr ordermap = NULL;
int ret = -1;
if (!(ordermap = virBitmapNew(maxvcpus)))
goto cleanup;
/* validate:
* - all hotpluggable entities to be hotplugged have the correct data
* - vcpus belonging to a hotpluggable entity share configuration
* - order of the hotpluggable entities is unique
*/
for (i = 0; i < maxvcpus; i++) {
vcpu = virDomainDefGetVcpu(def, i);
vcpupriv = QEMU_DOMAIN_VCPU_PRIVATE(vcpu);
/* skip over hotpluggable entities */
if (vcpupriv->vcpus == 0)
continue;
if (vcpu->order != 0) {
if (virBitmapIsBitSet(ordermap, vcpu->order - 1)) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
_("duplicate vcpu order '%u'"), vcpu->order - 1);
goto cleanup;
}
ignore_value(virBitmapSetBit(ordermap, vcpu->order - 1));
}
for (j = i + 1; j < (i + vcpupriv->vcpus); j++) {
subvcpu = virDomainDefGetVcpu(def, j);
if (subvcpu->hotpluggable != vcpu->hotpluggable ||
subvcpu->online != vcpu->online ||
subvcpu->order != vcpu->order) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
_("vcpus '%zu' and '%zu' are in the same hotplug "
"group but differ in configuration"), i, j);
goto cleanup;
}
}
if (vcpu->online && vcpu->hotpluggable == VIR_TRISTATE_BOOL_YES) {
if ((vcpupriv->socket_id == -1 && vcpupriv->core_id == -1 &&
vcpupriv->thread_id == -1) ||
!vcpupriv->type) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
_("vcpu '%zu' is missing hotplug data"), i);
goto cleanup;
}
}
}
ret = 0;
cleanup:
virBitmapFree(ordermap);
return ret;
}
static int
qemuDomainHasHotpluggableStartupVcpus(virDomainDefPtr def)
{
size_t maxvcpus = virDomainDefGetVcpusMax(def);
virDomainVcpuDefPtr vcpu;
size_t i;
for (i = 0; i < maxvcpus; i++) {
vcpu = virDomainDefGetVcpu(def, i);
if (vcpu->online && vcpu->hotpluggable == VIR_TRISTATE_BOOL_YES)
return true;
}
return false;
}
static int
qemuProcessVcpusSortOrder(const void *a,
const void *b)
{
virDomainVcpuDefPtr vcpua = *((virDomainVcpuDefPtr *)a);
virDomainVcpuDefPtr vcpub = *((virDomainVcpuDefPtr *)b);
return vcpua->order - vcpub->order;
}
static int
qemuProcessSetupHotpluggableVcpus(virQEMUDriverPtr driver,
virDomainObjPtr vm,
qemuDomainAsyncJob asyncJob)
{
unsigned int maxvcpus = virDomainDefGetVcpusMax(vm->def);
virDomainVcpuDefPtr vcpu;
qemuDomainVcpuPrivatePtr vcpupriv;
virJSONValuePtr vcpuprops = NULL;
size_t i;
int ret = -1;
int rc;
virDomainVcpuDefPtr *bootHotplug = NULL;
size_t nbootHotplug = 0;
for (i = 0; i < maxvcpus; i++) {
vcpu = virDomainDefGetVcpu(vm->def, i);
vcpupriv = QEMU_DOMAIN_VCPU_PRIVATE(vcpu);
if (vcpu->hotpluggable == VIR_TRISTATE_BOOL_YES && vcpu->online &&
vcpupriv->vcpus != 0) {
if (virAsprintf(&vcpupriv->alias, "vcpu%zu", i) < 0)
goto cleanup;
if (VIR_APPEND_ELEMENT(bootHotplug, nbootHotplug, vcpu) < 0)
goto cleanup;
}
}
if (nbootHotplug == 0) {
ret = 0;
goto cleanup;
}
qsort(bootHotplug, nbootHotplug, sizeof(*bootHotplug),
qemuProcessVcpusSortOrder);
for (i = 0; i < nbootHotplug; i++) {
vcpu = bootHotplug[i];
if (!(vcpuprops = qemuBuildHotpluggableCPUProps(vcpu)))
goto cleanup;
if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
goto cleanup;
rc = qemuMonitorAddDeviceArgs(qemuDomainGetMonitor(vm), vcpuprops);
vcpuprops = NULL;
if (qemuDomainObjExitMonitor(driver, vm) < 0)
goto cleanup;
if (rc < 0)
goto cleanup;
virJSONValueFree(vcpuprops);
}
ret = 0;
cleanup:
VIR_FREE(bootHotplug);
virJSONValueFree(vcpuprops);
return ret;
}
/**
* qemuProcessPrepareDomain
*
@ -5236,6 +5402,18 @@ qemuProcessLaunch(virConnectPtr conn,
if (qemuSetupCpusetMems(vm) < 0)
goto cleanup;
VIR_DEBUG("setting up hotpluggable cpus");
if (qemuDomainHasHotpluggableStartupVcpus(vm->def)) {
if (qemuDomainRefreshVcpuInfo(driver, vm, asyncJob, false) < 0)
goto cleanup;
if (qemuProcessValidateHotpluggableVcpus(vm->def) < 0)
goto cleanup;
if (qemuProcessSetupHotpluggableVcpus(driver, vm, asyncJob) < 0)
goto cleanup;
}
VIR_DEBUG("Refreshing VCPU info");
if (qemuDomainRefreshVcpuInfo(driver, vm, asyncJob, false) < 0)
goto cleanup;

20
tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.args Normal file
View File

@ -0,0 +1,20 @@
LC_ALL=C \
PATH=/bin \
HOME=/home/test \
USER=test \
LOGNAME=test \
QEMU_AUDIO_DRV=none \
/usr/bin/qemu \
-name QEMUGuest1 \
-S \
-M pc \
-m 214 \
-smp 1,maxcpus=6,sockets=3,cores=2,threads=1 \
-uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
-nographic \
-nodefaults \
-monitor unix:/tmp/lib/domain--1-QEMUGuest1/monitor.sock,server,nowait \
-no-acpi \
-boot n \
-usb \
-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3

29
tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.xml Normal file
View File

@ -0,0 +1,29 @@
<domain type='qemu'>
<name>QEMUGuest1</name>
<uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
<memory unit='KiB'>219100</memory>
<currentMemory unit='KiB'>219100</currentMemory>
<vcpu placement='static' current='3'>6</vcpu>
<vcpus>
<vcpu id='0' enabled='yes' hotpluggable='no' order='1'/>
<vcpu id='1' enabled='no' hotpluggable='yes'/>
<vcpu id='2' enabled='no' hotpluggable='yes'/>
<vcpu id='3' enabled='no' hotpluggable='yes'/>
<vcpu id='4' enabled='yes' hotpluggable='yes' order='2'/>
<vcpu id='5' enabled='yes' hotpluggable='yes' order='3'/>
</vcpus>
<os>
<type arch='x86_64' machine='pc'>hvm</type>
<boot dev='network'/>
</os>
<cpu>
<topology sockets="3" cores="2" threads="1"/>
</cpu>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<devices>
<emulator>/usr/bin/qemu</emulator>
</devices>
</domain>

2
tests/qemuxml2argvtest.c
View File

@ -2106,6 +2106,8 @@ mymain(void)
DO_TEST("intel-iommu", QEMU_CAPS_DEVICE_PCI_BRIDGE,
QEMU_CAPS_DEVICE_DMI_TO_PCI_BRIDGE, QEMU_CAPS_DEVICE_INTEL_IOMMU);
DO_TEST("cpu-hotplug-startup", QEMU_CAPS_QUERY_HOTPLUGGABLE_CPUS);
qemuTestDriverFree(&driver);
return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE;