From 97010eb1f14dfe705d58bf94ad4eec2cb293f560 Mon Sep 17 00:00:00 2001
From: Osier Yang <jyang@redhat.com>
Date: Wed, 9 May 2012 00:04:34 +0800
Subject: [PATCH] numad: Set memory policy from numad advisory nodeset

Though numad will manage the memory allocation of task dynamically,
it wants management application (libvirt) to pre-set the memory
policy according to the advisory nodeset returned from querying numad,
(just like pre-bind CPU nodeset for domain process), and thus the
performance could benefit much more from it.

This patch introduces new XML tag 'placement', value 'auto' indicates
whether to set the memory policy with the advisory nodeset from numad,
and its value defaults to the value of <vcpu> placement, or 'static'
if 'nodeset' is specified. Example of the new XML tag's usage:

  <numatune>
    <memory placement='auto' mode='interleave'/>
  </numatune>

Just like what current "numatune" does, the 'auto' numa memory policy
setting uses libnuma's API too.

If <vcpu> "placement" is "auto", and <numatune> is not specified
explicitly, a default <numatume> will be added with "placement"
set as "auto", and "mode" set as "strict".

The following XML can now fully drive numad:

1) <vcpu> placement is 'auto', no <numatune> is specified.

   <vcpu placement='auto'>10</vcpu>

2) <vcpu> placement is 'auto', no 'placement' is specified for
   <numatune>.

   <vcpu placement='auto'>10</vcpu>
   <numatune>
     <memory mode='interleave'/>
   </numatune>

And it's also able to control the CPU placement and memory policy
independently. e.g.

1) <vcpu> placement is 'auto', and <numatune> placement is 'static'

   <vcpu placement='auto'>10</vcpu>
   <numatune>
     <memory mode='strict' nodeset='0-10,^7'/>
   </numatune>

2) <vcpu> placement is 'static', and <numatune> placement is 'auto'

   <vcpu placement='static' cpuset='0-24,^12'>10</vcpu>
   <numatune>
     <memory mode='interleave' placement='auto'/>
   </numatume>

A follow up patch will change the XML formatting codes to always output
'placement' for <vcpu>, even it's 'static'.
---
 docs/formatdomain.html.in                     |  39 +++--
 docs/schemas/domaincommon.rng                 |  36 +++--
 src/conf/domain_conf.c                        | 133 +++++++++++++-----
 src/conf/domain_conf.h                        |  10 ++
 src/libvirt_private.syms                      |   2 +
 src/qemu/qemu_process.c                       |  85 ++++++-----
 ...muxml2argv-numad-auto-vcpu-no-numatune.xml |  29 ++++
 ...2argv-numad-auto-vcpu-static-numatune.args |   4 +
 ...l2argv-numad-auto-vcpu-static-numatune.xml |  31 ++++
 ...xml2argv-numad-static-vcpu-no-numatune.xml |  29 ++++
 .../qemuxml2argvdata/qemuxml2argv-numad.args  |   4 +
 tests/qemuxml2argvdata/qemuxml2argv-numad.xml |  31 ++++
 tests/qemuxml2argvtest.c                      |   2 +
 ...xml2xmlout-numad-auto-vcpu-no-numatune.xml |  32 +++++
 ...l2xmlout-numad-static-vcpu-no-numatune.xml |  29 ++++
 tests/qemuxml2xmltest.c                       |   2 +
 16 files changed, 404 insertions(+), 94 deletions(-)
 create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-no-numatune.xml
 create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-static-numatune.args
 create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-static-numatune.xml
 create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-numad-static-vcpu-no-numatune.xml
 create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-numad.args
 create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-numad.xml
 create mode 100644 tests/qemuxml2xmloutdata/qemuxml2xmlout-numad-auto-vcpu-no-numatune.xml
 create mode 100644 tests/qemuxml2xmloutdata/qemuxml2xmlout-numad-static-vcpu-no-numatune.xml

diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in
index 1478832433..da61312ce4 100644
--- a/docs/formatdomain.html.in
+++ b/docs/formatdomain.html.in
@@ -362,14 +362,14 @@
         0.9.11 (QEMU and KVM only)</span>, the optional attribute
         <code>placement</code> can be used to indicate the CPU placement
         mode for domain process, its value can be either "static" or
-        "auto", defaults to "static" if <code>cpuset</code> is specified,
-        "auto" indicates the domain process will be pinned to the advisory
-        nodeset from querying numad, and the value of attribute
-        <code>cpuset</code> will be ignored if it's specified. If both
-        <code>cpuset</code> and <code>placement</code> are not specified,
-        or if <code>placement</code> is "static", but no <code>cpuset</code>
-        is specified, the domain process will be pinned to all the
-        available physical CPUs.
+        "auto", defaults to <code>placement</code> of <code>numatune</code>,
+         or "static" if <code>cpuset</code> is specified. "auto" indicates
+        the domain process will be pinned to the advisory nodeset from querying
+        numad, and the value of attribute <code>cpuset</code> will be ignored
+        if it's specified. If both <code>cpuset</code> and <code>placement</code>
+        are not specified, or if <code>placement</code> is "static", but no
+        <code>cpuset</code> is specified, the domain process will be pinned to
+        all the available physical CPUs.
       </dd>
     </dl>
 
@@ -578,11 +578,24 @@
       <dt><code>memory</code></dt>
       <dd>
         The optional <code>memory</code> element specifies how to allocate memory
-        for the domain process on a NUMA host. It contains two attributes,
-        attribute <code>mode</code> is either 'interleave', 'strict',
-        or 'preferred',
-        attribute <code>nodeset</code> specifies the NUMA nodes, it leads same
-        syntax with attribute <code>cpuset</code> of element <code>vcpu</code>.
+        for the domain process on a NUMA host. It contains several optional
+        attributes. Attribute <code>mode</code> is either 'interleave',
+        'strict', or 'preferred', defaults to 'strict'. Attribute
+        <code>nodeset</code> specifies the NUMA nodes, using the same syntax as
+        attribute <code>cpuset</code> of element <code>vcpu</code>. Attribute
+        <code>placement</code> (<span class='since'>since 0.9.12</span>) can be
+        used to indicate the memory placement mode for domain process, its value
+        can be either "static" or "auto", defaults to <code>placement</code> of
+        <code>vcpu</code>, or "static" if <code>nodeset</code> is specified.
+        "auto" indicates the domain process will only allocate memory from the
+        advisory nodeset returned from querying numad, and the value of attribute
+        <code>nodeset</code> will be ignored if it's specified.
+
+        If <code>placement</code> of <code>vcpu</code> is 'auto', and
+        <code>numatune</code> is not specified, a default <code>numatune</code>
+        with <code>placement</code> 'auto' and <code>mode</code> 'strict' will
+        be added implicitly.
+
         <span class='since'>Since 0.9.3</span>
       </dd>
     </dl>
diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng
index 84369c7db4..4c4fa6a6d0 100644
--- a/docs/schemas/domaincommon.rng
+++ b/docs/schemas/domaincommon.rng
@@ -562,16 +562,32 @@
         <element name="numatune">
           <optional>
             <element name="memory">
-              <attribute name="mode">
-                <choice>
-                  <value>strict</value>
-                  <value>preferred</value>
-                  <value>interleave</value>
-                </choice>
-              </attribute>
-              <attribute name="nodeset">
-                <ref name="cpuset"/>
-              </attribute>
+              <optional>
+                <attribute name="mode">
+                  <choice>
+                    <value>strict</value>
+                    <value>preferred</value>
+                    <value>interleave</value>
+                  </choice>
+                </attribute>
+              </optional>
+              <choice>
+                <group>
+                  <optional>
+                    <attribute name='placement'>
+                      <value>static</value>
+                    </attribute>
+                  </optional>
+                  <optional>
+                    <attribute name='nodeset'>
+                      <ref name='cpuset'/>
+                    </attribute>
+                  </optional>
+                </group>
+                <attribute name='placement'>
+                  <value>auto</value>
+                </attribute>
+              </choice>
             </element>
           </optional>
         </element>
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index a60ef5a63e..f684ea1649 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -641,6 +641,12 @@ VIR_ENUM_IMPL(virDomainDiskTray, VIR_DOMAIN_DISK_TRAY_LAST,
               "closed",
               "open");
 
+VIR_ENUM_IMPL(virDomainNumatuneMemPlacementMode,
+              VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_LAST,
+              "default",
+              "static",
+              "auto");
+
 #define virDomainReportError(code, ...)                              \
     virReportErrorHelper(VIR_FROM_DOMAIN, code, __FILE__,            \
                          __FUNCTION__, __LINE__, __VA_ARGS__)
@@ -7969,9 +7975,9 @@ static virDomainDefPtr virDomainDefParseXML(virCapsPtr caps,
             if (virDomainCpuSetParse(set, 0, def->cpumask,
                                      def->cpumasklen) < 0)
                 goto error;
-            VIR_FREE(tmp);
             if (def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_DEFAULT)
                 def->placement_mode = VIR_DOMAIN_CPU_PLACEMENT_MODE_STATIC;
+            VIR_FREE(tmp);
         }
     }
 
@@ -8040,10 +8046,29 @@ static virDomainDefPtr virDomainDefParseXML(virCapsPtr caps,
         while (cur != NULL) {
             if (cur->type == XML_ELEMENT_NODE) {
                 if (xmlStrEqual(cur->name, BAD_CAST "memory")) {
-                    tmp = virXMLPropString(cur, "nodeset");
+                    char *mode = NULL;
+                    char *placement = NULL;
+                    char *nodeset = NULL;
 
-                    if (tmp) {
-                        char *set = tmp;
+                    mode = virXMLPropString(cur, "mode");
+                    if (mode) {
+                        if ((def->numatune.memory.mode =
+                             virDomainNumatuneMemModeTypeFromString(mode)) < 0) {
+                            virDomainReportError(VIR_ERR_XML_ERROR,
+                                                 _("Unsupported NUMA memory "
+                                                   "tuning mode '%s'"),
+                                                 mode);
+                            VIR_FREE(mode);
+                            goto error;
+                        }
+                        VIR_FREE(mode);
+                    } else {
+                        def->numatune.memory.mode = VIR_DOMAIN_NUMATUNE_MEM_STRICT;
+                    }
+
+                    nodeset = virXMLPropString(cur, "nodeset");
+                    if (nodeset) {
+                        char *set = nodeset;
                         int nodemasklen = VIR_DOMAIN_CPUMASK_LEN;
 
                         if (VIR_ALLOC_N(def->numatune.memory.nodemask,
@@ -8052,33 +8077,54 @@ static virDomainDefPtr virDomainDefParseXML(virCapsPtr caps,
                             goto error;
                         }
 
-                        /* "nodeset" leads same syntax with "cpuset". */
+                        /* "nodeset" uses the same syntax as "cpuset". */
                         if (virDomainCpuSetParse(set, 0,
                                                  def->numatune.memory.nodemask,
-                                                 nodemasklen) < 0)
+                                                 nodemasklen) < 0) {
+                            VIR_FREE(nodeset);
                             goto error;
-                        VIR_FREE(tmp);
+                        }
+                        VIR_FREE(nodeset);
+                    }
+
+                    placement = virXMLPropString(cur, "placement");
+                    int placement_mode = 0;
+                    if (placement) {
+                        if ((placement_mode =
+                             virDomainNumatuneMemPlacementModeTypeFromString(placement)) < 0) {
+                            virDomainReportError(VIR_ERR_XML_ERROR,
+                                                 _("Unsupported memory placement "
+                                                   "mode '%s'"), placement);
+                            VIR_FREE(placement);
+                            goto error;
+                        }
+                        VIR_FREE(placement);
+                    } else if (def->numatune.memory.nodemask) {
+                        /* Defaults to "static" if nodeset is specified. */
+                        placement_mode = VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC;
                     } else {
+                        /* Defaults to "placement" of <vcpu> if nodeset is
+                         * not specified.
+                         */
+                        if (def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_STATIC)
+                            placement_mode = VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC;
+                        else
+                            placement_mode = VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO;
+                    }
+
+                    if (placement_mode == VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC &&
+                        !def->numatune.memory.nodemask) {
                         virDomainReportError(VIR_ERR_XML_ERROR, "%s",
-                                             _("nodeset for NUMA memory "
-                                               "tuning must be set"));
+                                             _("nodeset for NUMA memory tuning must be set "
+                                               "if 'placement' is 'static'"));
                         goto error;
                     }
 
-                    tmp = virXMLPropString(cur, "mode");
-                    if (tmp) {
-                        if ((def->numatune.memory.mode =
-                            virDomainNumatuneMemModeTypeFromString(tmp)) < 0) {
-                            virDomainReportError(VIR_ERR_XML_ERROR,
-                                                 _("Unsupported NUMA memory "
-                                                   "tuning mode '%s'"),
-                                                 tmp);
-                            goto error;
-                        }
-                        VIR_FREE(tmp);
-                    } else {
-                        def->numatune.memory.mode = VIR_DOMAIN_NUMATUNE_MEM_STRICT;
-                    }
+                    /* Ignore 'nodeset' if 'placement' is 'auto' finally */
+                    if (placement_mode == VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)
+                        VIR_FREE(def->numatune.memory.nodemask);
+
+                    def->numatune.memory.placement_mode = placement_mode;
                 } else {
                     virDomainReportError(VIR_ERR_XML_ERROR,
                                          _("unsupported XML element %s"),
@@ -8088,6 +8134,14 @@ static virDomainDefPtr virDomainDefParseXML(virCapsPtr caps,
             }
             cur = cur->next;
         }
+    } else {
+        /* Defaults NUMA memory placement mode to 'auto' if no <numatune>
+         * and 'placement' of <vcpu> is 'auto'.
+         */
+        if (def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) {
+            def->numatune.memory.placement_mode = VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO;
+            def->numatune.memory.mode = VIR_DOMAIN_NUMATUNE_MEM_STRICT;
+        }
     }
     VIR_FREE(nodes);
 
@@ -12512,25 +12566,32 @@ virDomainDefFormatInternal(virDomainDefPtr def,
         def->cputune.period || def->cputune.quota)
         virBufferAddLit(buf, "  </cputune>\n");
 
-    if (def->numatune.memory.nodemask) {
+    if (def->numatune.memory.nodemask ||
+        def->numatune.memory.placement_mode) {
         virBufferAddLit(buf, "  <numatune>\n");
         const char *mode;
         char *nodemask = NULL;
-
-        nodemask = virDomainCpuSetFormat(def->numatune.memory.nodemask,
-                                         VIR_DOMAIN_CPUMASK_LEN);
-        if (nodemask == NULL) {
-            virDomainReportError(VIR_ERR_INTERNAL_ERROR, "%s",
-                                 _("failed to format nodeset for "
-                                   "NUMA memory tuning"));
-            goto cleanup;
-        }
+        const char *placement;
 
         mode = virDomainNumatuneMemModeTypeToString(def->numatune.memory.mode);
-        virBufferAsprintf(buf, "    <memory mode='%s' nodeset='%s'/>\n",
-                              mode, nodemask);
-        VIR_FREE(nodemask);
+        virBufferAsprintf(buf, "    <memory mode='%s' ", mode);
 
+        if (def->numatune.memory.placement_mode ==
+            VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) {
+            nodemask = virDomainCpuSetFormat(def->numatune.memory.nodemask,
+                                         VIR_DOMAIN_CPUMASK_LEN);
+            if (nodemask == NULL) {
+                virDomainReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                                     _("failed to format nodeset for "
+                                       "NUMA memory tuning"));
+                goto cleanup;
+            }
+            virBufferAsprintf(buf, "nodeset='%s'/>\n", nodemask);
+            VIR_FREE(nodemask);
+        } else if (def->numatune.memory.placement_mode) {
+            placement = virDomainNumatuneMemPlacementModeTypeToString(def->numatune.memory.placement_mode);
+            virBufferAsprintf(buf, "placement='%s'/>\n", placement);
+        }
         virBufferAddLit(buf, "  </numatune>\n");
     }
 
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 00178e1ffa..ecbdeaa378 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -1418,6 +1418,14 @@ enum virDomainCpuPlacementMode {
     VIR_DOMAIN_CPU_PLACEMENT_MODE_LAST,
 };
 
+enum virDomainNumatuneMemPlacementMode {
+    VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_DEFAULT = 0,
+    VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC,
+    VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO,
+
+    VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_LAST,
+};
+
 typedef struct _virDomainTimerCatchupDef virDomainTimerCatchupDef;
 typedef virDomainTimerCatchupDef *virDomainTimerCatchupDefPtr;
 struct _virDomainTimerCatchupDef {
@@ -1506,6 +1514,7 @@ struct _virDomainNumatuneDef {
     struct {
         char *nodemask;
         int mode;
+        int placement_mode; /* enum virDomainNumatuneMemPlacementMode */
     } memory;
 
     /* Future NUMA tuning related stuff should go here. */
@@ -2178,6 +2187,7 @@ VIR_ENUM_DECL(virDomainGraphicsSpiceStreamingMode)
 VIR_ENUM_DECL(virDomainGraphicsSpiceClipboardCopypaste)
 VIR_ENUM_DECL(virDomainGraphicsSpiceMouseMode)
 VIR_ENUM_DECL(virDomainNumatuneMemMode)
+VIR_ENUM_DECL(virDomainNumatuneMemPlacementMode)
 VIR_ENUM_DECL(virDomainSnapshotState)
 /* from libvirt.h */
 VIR_ENUM_DECL(virDomainState)
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 4f34d25610..afb308d901 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -400,6 +400,8 @@ virDomainNostateReasonTypeFromString;
 virDomainNostateReasonTypeToString;
 virDomainNumatuneMemModeTypeFromString;
 virDomainNumatuneMemModeTypeToString;
+virDomainNumatuneMemPlacementModeTypeFromString;
+virDomainNumatuneMemPlacementModeTypeToString;
 virDomainObjAssignDef;
 virDomainObjCopyPersistentDef;
 virDomainObjGetPersistentDef;
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 0355f8faae..82b17d6dce 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -1657,7 +1657,8 @@ qemuProcessDetectVcpuPIDs(struct qemud_driver *driver,
  */
 #if HAVE_NUMACTL
 static int
-qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm)
+qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm,
+                                const char *nodemask)
 {
     nodemask_t mask;
     int mode = -1;
@@ -1666,11 +1667,22 @@ qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm)
     int i = 0;
     int maxnode = 0;
     bool warned = false;
+    virDomainNumatuneDef numatune = vm->def->numatune;
+    const char *tmp_nodemask = NULL;
 
-    if (!vm->def->numatune.memory.nodemask)
+    if (numatune.memory.placement_mode ==
+        VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) {
+        if (!numatune.memory.nodemask)
+            return 0;
+        VIR_DEBUG("Set NUMA memory policy with specified nodeset");
+        tmp_nodemask = numatune.memory.nodemask;
+    } else if (numatune.memory.placement_mode ==
+               VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) {
+        VIR_DEBUG("Set NUMA memory policy with advisory nodeset from numad");
+        tmp_nodemask = nodemask;
+    } else {
         return 0;
-
-    VIR_DEBUG("Setting NUMA memory policy");
+    }
 
     if (numa_available() < 0) {
         qemuReportError(VIR_ERR_INTERNAL_ERROR,
@@ -1679,11 +1691,10 @@ qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm)
     }
 
     maxnode = numa_max_node() + 1;
-
     /* Convert nodemask to NUMA bitmask. */
     nodemask_zero(&mask);
     for (i = 0; i < VIR_DOMAIN_CPUMASK_LEN; i++) {
-        if (vm->def->numatune.memory.nodemask[i]) {
+        if (tmp_nodemask[i]) {
             if (i > NUMA_NUM_NODES) {
                 qemuReportError(VIR_ERR_INTERNAL_ERROR,
                                 _("Host cannot support NUMA node %d"), i);
@@ -1693,12 +1704,12 @@ qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm)
                 VIR_WARN("nodeset is out of range, there is only %d NUMA "
                          "nodes on host", maxnode);
                 warned = true;
-             }
+            }
             nodemask_set(&mask, i);
         }
     }
 
-    mode = vm->def->numatune.memory.mode;
+    mode = numatune.memory.mode;
 
     if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) {
         numa_set_bind_policy(1);
@@ -1789,7 +1800,8 @@ qemuGetNumadAdvice(virDomainDefPtr def ATTRIBUTE_UNUSED)
  */
 static int
 qemuProcessInitCpuAffinity(struct qemud_driver *driver,
-                           virDomainObjPtr vm)
+                           virDomainObjPtr vm,
+                           const char *nodemask)
 {
     int ret = -1;
     int i, hostcpus, maxcpu = QEMUD_CPUMASK_LEN;
@@ -1815,27 +1827,7 @@ qemuProcessInitCpuAffinity(struct qemud_driver *driver,
     }
 
     if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) {
-        char *nodeset = NULL;
-        char *nodemask = NULL;
-
-        nodeset = qemuGetNumadAdvice(vm->def);
-        if (!nodeset)
-            goto cleanup;
-
-        if (VIR_ALLOC_N(nodemask, VIR_DOMAIN_CPUMASK_LEN) < 0) {
-            virReportOOMError();
-            VIR_FREE(nodeset);
-            goto cleanup;
-        }
-
-        if (virDomainCpuSetParse(nodeset, 0, nodemask,
-                                 VIR_DOMAIN_CPUMASK_LEN) < 0) {
-            VIR_FREE(nodemask);
-            VIR_FREE(nodeset);
-            goto cleanup;
-        }
-        VIR_FREE(nodeset);
-
+        VIR_DEBUG("Set CPU affinity with advisory nodeset from numad");
         /* numad returns the NUMA node list, convert it to cpumap */
         int prev_total_ncpus = 0;
         for (i = 0; i < driver->caps->host.nnumaCell; i++) {
@@ -1852,9 +1844,8 @@ qemuProcessInitCpuAffinity(struct qemud_driver *driver,
             }
             prev_total_ncpus += cur_ncpus;
         }
-
-        VIR_FREE(nodemask);
     } else {
+        VIR_DEBUG("Set CPU affinity with specified cpuset");
         if (vm->def->cpumask) {
             /* XXX why don't we keep 'cpumask' in the libvirt cpumap
              * format to start with ?!?! */
@@ -2564,6 +2555,8 @@ static int qemuProcessHook(void *data)
     struct qemuProcessHookData *h = data;
     int ret = -1;
     int fd;
+    char *nodeset = NULL;
+    char *nodemask = NULL;
 
     /* Some later calls want pid present */
     h->vm->pid = getpid();
@@ -2597,14 +2590,34 @@ static int qemuProcessHook(void *data)
     if (qemuAddToCgroup(h->driver, h->vm->def) < 0)
         goto cleanup;
 
+    if ((h->vm->def->placement_mode ==
+         VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) ||
+        (h->vm->def->numatune.memory.placement_mode ==
+         VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)) {
+        nodeset = qemuGetNumadAdvice(h->vm->def);
+        if (!nodeset)
+            goto cleanup;
+
+        VIR_DEBUG("Nodeset returned from numad: %s", nodeset);
+
+        if (VIR_ALLOC_N(nodemask, VIR_DOMAIN_CPUMASK_LEN) < 0) {
+            virReportOOMError();
+            goto cleanup;
+        }
+
+        if (virDomainCpuSetParse(nodeset, 0, nodemask,
+                                 VIR_DOMAIN_CPUMASK_LEN) < 0)
+            goto cleanup;
+    }
+
     /* This must be done after cgroup placement to avoid resetting CPU
      * affinity */
     VIR_DEBUG("Setup CPU affinity");
-    if (qemuProcessInitCpuAffinity(h->driver, h->vm) < 0)
+    if (qemuProcessInitCpuAffinity(h->driver, h->vm, nodemask) < 0)
         goto cleanup;
 
-    if (qemuProcessInitNumaMemoryPolicy(h->vm) < 0)
-        return -1;
+    if (qemuProcessInitNumaMemoryPolicy(h->vm, nodemask) < 0)
+        goto cleanup;
 
     VIR_DEBUG("Setting up security labelling");
     if (virSecurityManagerSetProcessLabel(h->driver->securityManager, h->vm->def) < 0)
@@ -2614,6 +2627,8 @@ static int qemuProcessHook(void *data)
 
 cleanup:
     VIR_DEBUG("Hook complete ret=%d", ret);
+    VIR_FREE(nodeset);
+    VIR_FREE(nodemask);
     return ret;
 }
 
diff --git a/tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-no-numatune.xml b/tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-no-numatune.xml
new file mode 100644
index 0000000000..175f3fb93d
--- /dev/null
+++ b/tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-no-numatune.xml
@@ -0,0 +1,29 @@
+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='auto'>2</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <cpu>
+    <topology sockets='2' cores='1' threads='1'/>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='ide' index='0'/>
+    <controller type='usb' index='0'/>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>
diff --git a/tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-static-numatune.args b/tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-static-numatune.args
new file mode 100644
index 0000000000..23bcb70000
--- /dev/null
+++ b/tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-static-numatune.args
@@ -0,0 +1,4 @@
+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test /usr/bin/qemu -S -M \
+pc -m 214 -smp 2 -nographic -monitor \
+unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -hda \
+/dev/HostVG/QEMUGuest1 -net none -serial none -parallel none -usb
diff --git a/tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-static-numatune.xml b/tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-static-numatune.xml
new file mode 100644
index 0000000000..c5757f48b4
--- /dev/null
+++ b/tests/qemuxml2argvdata/qemuxml2argv-numad-auto-vcpu-static-numatune.xml
@@ -0,0 +1,31 @@
+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='auto'>2</vcpu>
+  <numatune>
+    <memory mode='interleave' nodeset='0'/>
+  </numatune>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <cpu>
+    <topology sockets='2' cores='1' threads='1'/>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='ide' index='0'/>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>
diff --git a/tests/qemuxml2argvdata/qemuxml2argv-numad-static-vcpu-no-numatune.xml b/tests/qemuxml2argvdata/qemuxml2argv-numad-static-vcpu-no-numatune.xml
new file mode 100644
index 0000000000..71c1497aa9
--- /dev/null
+++ b/tests/qemuxml2argvdata/qemuxml2argv-numad-static-vcpu-no-numatune.xml
@@ -0,0 +1,29 @@
+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='static'>2</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <cpu>
+    <topology sockets='2' cores='1' threads='1'/>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='ide' index='0'/>
+    <controller type='usb' index='0'/>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>
diff --git a/tests/qemuxml2argvdata/qemuxml2argv-numad.args b/tests/qemuxml2argvdata/qemuxml2argv-numad.args
new file mode 100644
index 0000000000..23bcb70000
--- /dev/null
+++ b/tests/qemuxml2argvdata/qemuxml2argv-numad.args
@@ -0,0 +1,4 @@
+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test /usr/bin/qemu -S -M \
+pc -m 214 -smp 2 -nographic -monitor \
+unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -hda \
+/dev/HostVG/QEMUGuest1 -net none -serial none -parallel none -usb
diff --git a/tests/qemuxml2argvdata/qemuxml2argv-numad.xml b/tests/qemuxml2argvdata/qemuxml2argv-numad.xml
new file mode 100644
index 0000000000..c87ec49fba
--- /dev/null
+++ b/tests/qemuxml2argvdata/qemuxml2argv-numad.xml
@@ -0,0 +1,31 @@
+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='auto'>2</vcpu>
+  <numatune>
+    <memory mode="interleave" placement='auto'/>
+  </numatune>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <cpu>
+    <topology sockets='2' cores='1' threads='1'/>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='ide' index='0'/>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>
diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
index 4d7cf6972c..40471d4809 100644
--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@@ -738,6 +738,8 @@ mymain(void)
     DO_TEST("blkiotune-device", false, QEMU_CAPS_NAME);
     DO_TEST("cputune", false, QEMU_CAPS_NAME);
     DO_TEST("numatune-memory", false, NONE);
+    DO_TEST("numad", false, NONE);
+    DO_TEST("numad-auto-vcpu-static-numatune", false, NONE);
     DO_TEST("blkdeviotune", false, QEMU_CAPS_NAME, QEMU_CAPS_DEVICE,
             QEMU_CAPS_DRIVE, QEMU_CAPS_DRIVE_IOTUNE);
 
diff --git a/tests/qemuxml2xmloutdata/qemuxml2xmlout-numad-auto-vcpu-no-numatune.xml b/tests/qemuxml2xmloutdata/qemuxml2xmlout-numad-auto-vcpu-no-numatune.xml
new file mode 100644
index 0000000000..26de1b2f4d
--- /dev/null
+++ b/tests/qemuxml2xmloutdata/qemuxml2xmlout-numad-auto-vcpu-no-numatune.xml
@@ -0,0 +1,32 @@
+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='auto'>2</vcpu>
+  <numatune>
+    <memory mode='strict' placement='auto'/>
+  </numatune>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <cpu>
+    <topology sockets='2' cores='1' threads='1'/>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='ide' index='0'/>
+    <controller type='usb' index='0'/>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>
diff --git a/tests/qemuxml2xmloutdata/qemuxml2xmlout-numad-static-vcpu-no-numatune.xml b/tests/qemuxml2xmloutdata/qemuxml2xmlout-numad-static-vcpu-no-numatune.xml
new file mode 100644
index 0000000000..71c1497aa9
--- /dev/null
+++ b/tests/qemuxml2xmloutdata/qemuxml2xmlout-numad-static-vcpu-no-numatune.xml
@@ -0,0 +1,29 @@
+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='static'>2</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <cpu>
+    <topology sockets='2' cores='1' threads='1'/>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='ide' index='0'/>
+    <controller type='usb' index='0'/>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>
diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c
index 9bca066e64..256eed7e39 100644
--- a/tests/qemuxml2xmltest.c
+++ b/tests/qemuxml2xmltest.c
@@ -218,6 +218,7 @@ mymain(void)
     DO_TEST_FULL("seclabel-dynamic-override", false, WHEN_INACTIVE);
     DO_TEST("seclabel-static");
     DO_TEST("seclabel-none");
+    DO_TEST("numad-static-vcpu-no-numatune");
 
     /* These tests generate different XML */
     DO_TEST_DIFFERENT("balloon-device-auto");
@@ -228,6 +229,7 @@ mymain(void)
     DO_TEST_DIFFERENT("serial-target-port-auto");
     DO_TEST_DIFFERENT("graphics-listen-network2");
     DO_TEST_DIFFERENT("graphics-spice-timeout");
+    DO_TEST_DIFFERENT("numad-auto-vcpu-no-numatune");
 
     DO_TEST_DIFFERENT("metadata");