From e11dab257f25ef80769ea3568a7a289c754ea017 Mon Sep 17 00:00:00 2001 From: "Daniel P. Berrange" Date: Wed, 29 Apr 2015 18:25:11 +0100 Subject: [PATCH] docs: fixup libvirt NUMA testing docs to match reality The libvirt NUMA testing docs were written before the final code was merged, so the example JSON shown for the database fields did not match reality. It was also hard to read the JSON since there were no line breaks & indentation. To make matters even worse, the final example was just plain wrong - the data shown was a mixture of a guest with 1 NUMA node and a guest with 2 NUMA nodes. Change-Id: I51e4df2d9fe252ba65f6a37d84f125e18b44e239 --- doc/source/testing/libvirt-numa.rst | 309 +++++++++++++++++++++++++--- 1 file changed, 284 insertions(+), 25 deletions(-) diff --git a/doc/source/testing/libvirt-numa.rst b/doc/source/testing/libvirt-numa.rst index 2296244bad35..83e56191fbb3 100644 --- a/doc/source/testing/libvirt-numa.rst +++ b/doc/source/testing/libvirt-numa.rst @@ -182,7 +182,7 @@ operation of Nova libvirt guests boot a tiny instance .. code-block:: bash - # . openrc admin admin + # . openrc admin # nova boot --image cirros-0.3.2-x86_64-uec --flavor m1.tiny cirros1 The host will be reporting NUMA topology, but there should only @@ -192,11 +192,46 @@ be a single NUMA cell this point. # mysql -u root -p nova MariaDB [nova]> select numa_topology from compute_nodes; - +------------------------------------------------------------------------------------------------------+ - | numa_topology | - +------------------------------------------------------------------------------------------------------+ - | {"cells": [{"mem": {"total": 7794, "used": 0}, "cpu_usage": 0, "cpus": "0,1,2,3,4,5,6,7", "id": 0}]} | - +------------------------------------------------------------------------------------------------------+ + +----------------------------------------------------------------------------+ + | numa_topology | + +----------------------------------------------------------------------------+ + | { + | "nova_object.name": "NUMATopology", + | "nova_object.data": { + | "cells": [{ + | "nova_object.name": "NUMACell", + | "nova_object.data": { + | "cpu_usage": 0, + | "memory_usage": 0, + | "cpuset": [0, 1, 2, 3, 4, 5, 6, 7], + | "pinned_cpus": [], + | "siblings": [], + | "memory": 7793, + | "mempages": [ + | { + | "nova_object.name": "NUMAPagesTopology", + | "nova_object.data": { + | "total": 987430, + | "used": 0, + | "size_kb": 4 + | }, + | }, + | { + | "nova_object.name": "NUMAPagesTopology", + | "nova_object.data": { + | "total": 0, + | "used": 0, + | "size_kb": 2048 + | }, + | } + | ], + | "id": 0 + | }, + | }, + | ] + | }, + | } + +----------------------------------------------------------------------------+ Meanwhile, the guest instance should not have any NUMA configuration @@ -276,7 +311,7 @@ Set the following parameters: .. code-block:: bash [DEFAULT] - scheduler_default_filters=RetryFilter, AvailabilityZoneFilter, RamFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter, NUMATopologyFilter + scheduler_default_filters=RetryFilter, AvailabilityZoneFilter, RamFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, NUMATopologyFilter [libvirt] virt_type = kvm @@ -297,12 +332,106 @@ new NUMA topology setup for the guest # mysql -u root -p nova MariaDB [nova]> select numa_topology from compute_nodes; - +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - | numa_topology | - +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - | {"cells": [{"mem": {"total": 3857, "used": 0}, "cpu_usage": 0, "cpus": "0,1,2,3", "id": 0}, {"mem": {"total": 1969, "used": 0}, "cpu_usage": 0, "cpus": "4,5", "id": 1}, {"mem": {"total": 1967, "used": 0}, "cpu_usage": 0, "cpus": "6,7", "id": 2}]} | - +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - + +----------------------------------------------------------------------------+ + | numa_topology | + +----------------------------------------------------------------------------+ + | { + | "nova_object.name": "NUMATopology", + | "nova_object.data": { + | "cells": [{ + | "nova_object.name": "NUMACell", + | "nova_object.data": { + | "cpu_usage": 0, + | "memory_usage": 0, + | "cpuset": [0, 1, 2, 3], + | "pinned_cpus": [], + | "siblings": [], + | "memory": 3857, + | "mempages": [ + | { + | "nova_object.name": "NUMAPagesTopology", + | "nova_object.data": { + | "total": 987430, + | "used": 0, + | "size_kb": 4 + | }, + | }, + | { + | "nova_object.name": "NUMAPagesTopology", + | "nova_object.data": { + | "total": 0, + | "used": 0, + | "size_kb": 2048 + | }, + | } + | ], + | "id": 0 + | }, + | }, + | { + | "nova_object.name": "NUMACell", + | "nova_object.data": { + | "cpu_usage": 0, + | "memory_usage": 0, + | "cpuset": [4, 5], + | "pinned_cpus": [], + | "siblings": [], + | "memory": 1969, + | "mempages": [ + | { + | "nova_object.name": "NUMAPagesTopology", + | "nova_object.data": { + | "total": 504216, + | "used": 0, + | "size_kb": 4 + | }, + | }, + | { + | "nova_object.name": "NUMAPagesTopology", + | "nova_object.data": { + | "total": 0, + | "used": 0, + | "size_kb": 2048 + | }, + | } + | ], + | "id": 1 + | }, + | }, + | { + | "nova_object.name": "NUMACell", + | "nova_object.data": { + | "cpu_usage": 0, + | "memory_usage": 0, + | "cpuset": [6, 7], + | "pinned_cpus": [], + | "siblings": [], + | "memory": 1967, + | "mempages": [ + | { + | "nova_object.name": "NUMAPagesTopology", + | "nova_object.data": { + | "total": 503575, + | "used": 0, + | "size_kb": 4 + | }, + | }, + | { + | "nova_object.name": "NUMAPagesTopology", + | "nova_object.data": { + | "total": 0, + | "used": 0, + | "size_kb": 2048 + | }, + | } + | ], + | "id": 2 + | }, + | } + | ] + | }, + | } + +----------------------------------------------------------------------------+ This indeed shows that there are now 3 NUMA nodes for the "host" machine, the first with 4 GB of RAM and 4 CPUs, and others with @@ -380,15 +509,13 @@ Looking at the resulting guest XML from libvirt - - + ... - + - The XML shows: @@ -396,11 +523,11 @@ The XML shows: * Each guest CPU has been pinned to the physical CPUs associated with a particular NUMA node * The emulator threads have been pinned to the union - of all physical CPUs in the host NUMA nodes that + of all physical CPUs in the host NUMA node that the guest is placed on * The guest has been given a virtual NUMA topology - splitting RAM and CPUs symmetrically -* Each guest NUMA node has been strictly pinned to + with a single node holding all RAM and CPUs +* The guest NUMA node has been strictly pinned to a host NUMA node. As a further sanity test, check what Nova recorded for the @@ -410,8 +537,140 @@ information .. code-block:: bash MariaDB [nova]> select numa_topology from instance_extra; - +---------------------------------------------------------------------------------------------------------------+ - | numa_topology | - +---------------------------------------------------------------------------------------------------------------+ - | {"cells": [{"mem": {"total": 512}, "cpus": "0,1", "id": 0}, {"mem": {"total": 512}, "cpus": "2,3", "id": 1}]} | - +---------------------------------------------------------------------------------------------------------------+ + +----------------------------------------------------------------------------+ + | numa_topology | + +----------------------------------------------------------------------------+ + | { + | "nova_object.name": "InstanceNUMATopology", + | "nova_object.data": { + | "instance_uuid": "4c2302fe-3f0f-46f1-9f3e-244011f6e03a", + | "cells": [ + | { + | "nova_object.name": "InstanceNUMACell", + | "nova_object.data": { + | "cpu_topology": null, + | "pagesize": null, + | "cpuset": [ + | 0, + | 1, + | 2, + | 3 + | ], + | "memory": 1024, + | "cpu_pinning_raw": null, + | "id": 0 + | }, + | } + | ] + | }, + | } + +----------------------------------------------------------------------------+ + +------------------------------------------------- +Testing instance boot with 2 NUMA cells requested +------------------------------------------------- + +Now getting more advanced we tell Nova that the guest will have two +NUMA nodes. To define the topology we will change the previously +defined flavor + +.. code-block:: bash + + # nova flavor-key m1.numa set hw:numa_nodes=2 + # nova flavor-show m1.numa + +Now boot the guest using this changed flavor + +.. code-block:: bash + + # nova boot --image cirros-0.3.2-x86_64-uec --flavor m1.numa cirros2 + +Looking at the resulting guest XML from libvirt + +.. code-block:: bash + + # virsh -c qemu:///system dumpxml instanceXXXXXX + ... + 4 + + + + + + + + ... + + + + + + + + ... + + + + + + +The XML shows: + +* Each guest CPU has been pinned to the physical CPUs + associated with particular NUMA nodes +* The emulator threads have been pinned to the union + of all physical CPUs in the host NUMA nodes that + the guest is placed on +* The guest has been given a virtual NUMA topology + with two nodes, each holding half the RAM and CPUs +* The guest NUMA nodes have been strictly pinned to + different host NUMA node. + +As a further sanity test, check what Nova recorded for the +instance in the database. This should match the +information + +.. code-block:: bash + + MariaDB [nova]> select numa_topology from instance_extra; + +----------------------------------------------------------------------------+ + | numa_topology | + +----------------------------------------------------------------------------+ + | { + | "nova_object.name": "InstanceNUMATopology", + | "nova_object.data": { + | "instance_uuid": "a14fcd68-567e-4d71-aaa4-a12f23f16d14", + | "cells": [ + | { + | "nova_object.name": "InstanceNUMACell", + | "nova_object.data": { + | "cpu_topology": null, + | "pagesize": null, + | "cpuset": [ + | 0, + | 1 + | ], + | "memory": 512, + | "cpu_pinning_raw": null, + | "id": 0 + | }, + | }, + | { + | "nova_object.name": "InstanceNUMACell", + | "nova_object.data": { + | "cpu_topology": null, + | "pagesize": null, + | "cpuset": [ + | 2, + | 3 + | ], + | "memory": 512, + | "cpu_pinning_raw": null, + | "id": 1 + | }, + | } + | ] + | }, + | } + | + +----------------------------------------------------------------------------+