From 4719ec15e97d96befaaeecc7bd5c7bcdce53e53d Mon Sep 17 00:00:00 2001 From: Aleksei Zakharov Date: Fri, 19 Feb 2021 23:08:34 +0300 Subject: [PATCH] qemu: add per-vcpu delay stats This patch adds delay time (steal time inside guest) to libvirt domain per-vcpu stats. Delay time is an important performance metric. It is a consequence of the overloaded CPU. Knowledge of the delay time of a virtual machine helps to understand if it is affected and estimate the impact. As a result, it is possible to react exactly when needed and rebalance the load between hosts. This is used by cloud providers to provide quality of service, especially when the CPU is oversubscribed. It's more convenient to work with this metric in a context of a libvirt domain. Any monitoring software may use this information. Signed-off-by: Aleksei Zakharov Signed-off-by: Michal Privoznik Reviewed-by: Daniel Henrique Barboza Reviewed-by: Michal Privoznik --- docs/manpages/virsh.rst | 4 ++++ src/libvirt-domain.c | 4 ++++ src/qemu/qemu_driver.c | 47 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/manpages/virsh.rst b/docs/manpages/virsh.rst index d44c36e2e9..9ff31a0160 100644 --- a/docs/manpages/virsh.rst +++ b/docs/manpages/virsh.rst @@ -2295,6 +2295,10 @@ When selecting the *--state* group the following fields are returned: * ``vcpu..halted`` - virtual CPU is halted: yes or no (may indicate the processor is idle or even disabled, depending on the architecture) +* ``vcpu..delay`` - time the vCPU thread was enqueued by the + host scheduler, but was waiting in the queue instead of running. + Exposed to the VM as a steal time. + *--interface* returns: diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 4af0166872..e54d11e513 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -11693,6 +11693,10 @@ virConnectGetDomainCapabilities(virConnectPtr conn, * "vcpu..halted" - virtual CPU is halted, may indicate the * processor is idle or even disabled, depending * on the architecture) + * "vcpu..delay" - time the vCPU thread was enqueued by the + * host scheduler, but was waiting in the queue + * instead of running. Exposed to the VM as a steal + * time. * * VIR_DOMAIN_STATS_INTERFACE: * Return network interface statistics (from domain point of view). diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 185fe903b2..c7323e1fa5 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -1333,6 +1333,37 @@ static char *qemuConnectGetCapabilities(virConnectPtr conn) { } +static int +qemuGetSchedstatDelay(unsigned long long *cpudelay, + pid_t pid, + pid_t tid) +{ + g_autofree char *path = NULL; + g_autofree char *buf = NULL; + + if (tid) + path = g_strdup_printf("/proc/%d/task/%d/schedstat", (int)pid, (int)tid); + else + path = g_strdup_printf("/proc/%d/schedstat", (int)pid); + + /* This file might not exist (needs CONFIG_SCHED_INFO) */ + if (!virFileExists(path)) + return 0; + + if (virFileReadAll(path, 1024, &buf) < 0) + return -1; + + if (sscanf(buf, "%*u %llu", cpudelay) != 1) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unable to parse schedstat info at '%s'"), + path); + return -1; + } + + return 0; +} + + static int qemuGetSchedInfo(unsigned long long *cpuWait, pid_t pid, pid_t tid) @@ -1470,6 +1501,7 @@ static int qemuDomainHelperGetVcpus(virDomainObjPtr vm, virVcpuInfoPtr info, unsigned long long *cpuwait, + unsigned long long *cpudelay, int maxinfo, unsigned char *cpumaps, int maplen) @@ -1529,6 +1561,11 @@ qemuDomainHelperGetVcpus(virDomainObjPtr vm, return -1; } + if (cpudelay) { + if (qemuGetSchedstatDelay(&(cpudelay[ncpuinfo]), vm->pid, vcpupid) < 0) + return -1; + } + ncpuinfo++; } @@ -4873,7 +4910,7 @@ qemuDomainGetVcpus(virDomainPtr dom, goto cleanup; } - ret = qemuDomainHelperGetVcpus(vm, info, NULL, maxinfo, cpumaps, maplen); + ret = qemuDomainHelperGetVcpus(vm, info, NULL, NULL, maxinfo, cpumaps, maplen); cleanup: virDomainObjEndAPI(&vm); @@ -17928,6 +17965,7 @@ qemuDomainGetStatsVcpu(virQEMUDriverPtr driver, int ret = -1; virVcpuInfoPtr cpuinfo = NULL; g_autofree unsigned long long *cpuwait = NULL; + g_autofree unsigned long long *cpudelay = NULL; if (virTypedParamListAddUInt(params, virDomainDefGetVcpus(dom->def), "vcpu.current") < 0) @@ -17939,6 +17977,7 @@ qemuDomainGetStatsVcpu(virQEMUDriverPtr driver, cpuinfo = g_new0(virVcpuInfo, virDomainDefGetVcpus(dom->def)); cpuwait = g_new0(unsigned long long, virDomainDefGetVcpus(dom->def)); + cpudelay = g_new0(unsigned long long, virDomainDefGetVcpus(dom->def)); if (HAVE_JOB(privflags) && virDomainObjIsActive(dom) && qemuDomainRefreshVcpuHalted(driver, dom, QEMU_ASYNC_JOB_NONE) < 0) { @@ -17947,7 +17986,7 @@ qemuDomainGetStatsVcpu(virQEMUDriverPtr driver, virResetLastError(); } - if (qemuDomainHelperGetVcpus(dom, cpuinfo, cpuwait, + if (qemuDomainHelperGetVcpus(dom, cpuinfo, cpuwait, cpudelay, virDomainDefGetVcpus(dom->def), NULL, 0) < 0) { virResetLastError(); @@ -17972,6 +18011,10 @@ qemuDomainGetStatsVcpu(virQEMUDriverPtr driver, "vcpu.%u.wait", cpuinfo[i].number) < 0) goto cleanup; + if (virTypedParamListAddULLong(params, cpudelay[i], + "vcpu.%u.delay", cpuinfo[i].number) < 0) + goto cleanup; + /* state below is extracted from the individual vcpu structs */ if (!(vcpu = virDomainDefGetVcpu(dom->def, cpuinfo[i].number))) continue;