From a272143739bc9443fb034e8d76fb6452243baa91 Mon Sep 17 00:00:00 2001 From: Tao Liu Date: Tue, 11 Sep 2018 13:07:35 -0400 Subject: [PATCH] Update compute_reserved.conf via a platform manifest The /etc/nova/compute_reserved.conf is not updated by the generic platform module compute.pp manifest. This results in some of the values not being set properly for ovs-dpdk in starlingx. This update creates an ERB template for compute_reserved.conf and manages the ERB template through the platform compute.pp manifest. In addition, the hiera data generation has been added to sysinv puppet platform operator. Closes-Bug: #1789726 Change-Id: I3a005482342532229c66b5bd9f36d581a85d30f7 Signed-off-by: Tao Liu --- .../src/modules/platform/manifests/compute.pp | 20 +++++ .../templates/compute_reserved.conf.erb | 74 +++++++++++++++++++ .../sysinv/sysinv/sysinv/puppet/platform.py | 68 ++++++++++++++++- 3 files changed, 159 insertions(+), 3 deletions(-) create mode 100755 puppet-manifests/src/modules/platform/templates/compute_reserved.conf.erb diff --git a/puppet-manifests/src/modules/platform/manifests/compute.pp b/puppet-manifests/src/modules/platform/manifests/compute.pp index 4846432e6d..4c1c8c766e 100644 --- a/puppet-manifests/src/modules/platform/manifests/compute.pp +++ b/puppet-manifests/src/modules/platform/manifests/compute.pp @@ -1,3 +1,22 @@ +class platform::compute::params ( + $compute_cpu_list = '', + $platform_cpu_list = '', + $reserved_vswitch_cores = '', + $reserved_platform_cores = '', + $compute_base_reserved = '', + $compute_vswitch_reserved = '', +) { } + +class platform::compute::config + inherits ::platform::compute::params { + + file { "/etc/nova/compute_reserved.conf": + ensure => 'present', + replace => true, + content => template('platform/compute_reserved.conf.erb') + } +} + class platform::compute::grub::params ( $n_cpus = '', $cpu_options = '', @@ -243,4 +262,5 @@ class platform::compute { require ::platform::compute::pmqos require ::platform::compute::resctrl require ::platform::compute::extend + require ::platform::compute::config } diff --git a/puppet-manifests/src/modules/platform/templates/compute_reserved.conf.erb b/puppet-manifests/src/modules/platform/templates/compute_reserved.conf.erb new file mode 100755 index 0000000000..429fded4cf --- /dev/null +++ b/puppet-manifests/src/modules/platform/templates/compute_reserved.conf.erb @@ -0,0 +1,74 @@ +################################################################################ +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# - This file is managed by Puppet. DO NOT EDIT. +################################################################################ +# COMPUTE Node configuration parameters for reserved memory and physical cores +# used by Base software and VSWITCH. These are resources that libvirt cannot use. +# + +################################################################################ +# +# List of logical CPU instances available in the system. This value is used +# for auditing purposes so that the current configuration can be checked for +# validity against the actual number of logical CPU instances in the system. +# +################################################################################ +COMPUTE_CPU_LIST=<%= @compute_cpu_list %> + +################################################################################ +# +# List of logical CPU instances that reserved for platform applications. +# +################################################################################ +PLATFORM_CPU_LIST=<%= @platform_cpu_list %> + +################################################################################ +# +# List of Base software resources reserved per numa node. Each array element +# consists of a 3-tuple formatted as: ::. +# +# Example: To reserve 1500MB and 1 core on NUMA node0, and 1500MB and 1 core +# on NUMA node1, the variable must be specified as follows. +# COMPUTE_BASE_MEMORY=("node0:1500MB:1" "node1:1500MB:1") +# +################################################################################ +COMPUTE_BASE_RESERVED=<%= @compute_base_reserved %> + +################################################################################ +# +# List of HugeTLB memory descriptors to configure. Each array element +# consists of a 3-tuple descriptor formatted as: ::. +# The NUMA node specified must exist and the HugeTLB pagesize must be a valid +# value such as 2048kB or 1048576kB. +# +# For example, to request 256 x 2MB HugeTLB pages on NUMA node0 and node1 the +# variable must be specified as follows. +# COMPUTE_VSWITCH_MEMORY=("node0:2048kB:256" "node1:2048kB:256") +# +################################################################################ +COMPUTE_VSWITCH_MEMORY=<%= @compute_vswitch_reserved %> + +################################################################################ +# +# List of VSWITCH physical cores reserved for VSWITCH applications. +# +# Example: To reserve 2 cores on NUMA node0, and 2 cores on NUMA node1, the +# variable must be specified as follows. +# COMPUTE_VSWITCH_CORES=("node0:2" "node1:2") +# +################################################################################ +COMPUTE_VSWITCH_CORES=<%= @reserved_vswitch_cores %> + +################################################################################ +# +# List of platform physical cores reserved for platform applications. +# +# Example: To reserve 1 core on NUMA node0, the variable must be specified +# as follows. +# COMPUTE_PLATFORM_CORES=("node0:0") +# +################################################################################ +COMPUTE_PLATFORM_CORES=<%= @reserved_platform_cores %> diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/platform.py b/sysinv/sysinv/sysinv/sysinv/puppet/platform.py index a348481bce..9e24d11cb4 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/platform.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/platform.py @@ -7,6 +7,7 @@ import copy import itertools import os +import operator from sysinv.common import constants from sysinv.common import exception @@ -68,7 +69,7 @@ class PlatformPuppet(base.BasePuppet): config.update(self._get_host_upgrade_config(host)) config.update(self._get_host_tpm_config(host)) config.update(self._get_host_cpu_config(host)) - config.update(self._get_host_hugepage_config(host)) + config.update(self._get_host_memory_config(host)) return config def _get_static_software_config(self): @@ -530,6 +531,36 @@ class PlatformPuppet(base.BasePuppet): if not host_cpus: return config + # Define the full range of CPUs for the compute host + max_cpu = max(host_cpus, key=operator.attrgetter('cpu')) + compute_cpu_list = "\"0-%d\"" % max_cpu.cpu + + platform_cpus_no_threads = self._get_platform_cpu_list(host) + vswitch_cpus_no_threads = self._get_vswitch_cpu_list(host) + + platform_cpu_list_with_quotes = \ + "\"%s\"" % ','.join([str(c.cpu) for c in platform_cpus_no_threads]) + + platform_numa_cpus = self._get_numa_index_list(platform_cpus_no_threads) + vswitch_numa_cpus = self._get_numa_index_list(vswitch_cpus_no_threads) + + # build a list of platform reserved cpus per numa node + platform_cores = [] + for node, cpus in platform_numa_cpus.items(): + cpu_list = ','.join([str(c.cpu) for c in cpus]) + platform_node = "\"node%d:%s\"" % (node, cpu_list) + platform_cores.append(platform_node) + + # build a list of vswitch reserved cpu counts per numa node + vswitch_cores = [] + for node, cpus in vswitch_numa_cpus.items(): + cpu_count = len(cpus) + vswitch_node = "\"node%d:%d\"" % (node, cpu_count) + vswitch_cores.append(vswitch_node) + + reserved_platform_cores = "(%s)" % ' '.join(platform_cores) + reserved_vswitch_cores = "(%s)" % ' '.join(vswitch_cores) + host_cpus = sorted(host_cpus, key=lambda c: c.cpu) n_cpus = len(host_cpus) host_cpu_list = [c.cpu for c in host_cpus] @@ -589,18 +620,31 @@ class PlatformPuppet(base.BasePuppet): platform_cpu_list, platform_cpu_list) config.update({ + 'platform::compute::params::compute_cpu_list': + compute_cpu_list, + 'platform::compute::params::platform_cpu_list': + platform_cpu_list_with_quotes, + 'platform::compute::params::reserved_vswitch_cores': + reserved_vswitch_cores, + 'platform::compute::params::reserved_platform_cores': + reserved_platform_cores, 'platform::compute::grub::params::n_cpus': n_cpus, 'platform::compute::grub::params::cpu_options': cpu_options, }) return config - def _get_host_hugepage_config(self, host): + def _get_host_memory_config(self, host): config = {} if constants.COMPUTE in utils.get_personalities(host): host_memory = self.dbapi.imemory_get_by_ihost(host.id) - memory_numa_list = self._get_numa_index_list(host_memory) + platform_cpus = self._get_platform_cpu_list(host) + platform_cpu_count = len(platform_cpus) + + platform_nodes = [] + vswitch_nodes = [] + hugepages_2Ms = [] hugepages_1Gs = [] vswitch_2M_pages = [] @@ -615,6 +659,17 @@ class PlatformPuppet(base.BasePuppet): vswitch_2M_page = 0 vswitch_1G_page = 0 + platform_size = memory.platform_reserved_mib + platform_node = "\"node%d:%dMB:%d\"" % ( + node, platform_size, platform_cpu_count) + platform_nodes.append(platform_node) + + vswitch_size = memory.vswitch_hugepages_size_mib + vswitch_pages = memory.vswitch_hugepages_nr + vswitch_node = "\"node%d:%dkB:%d\"" % ( + node, vswitch_size * 1024, vswitch_pages) + vswitch_nodes.append(vswitch_node) + vm_hugepages_nr_2M = memory.vm_hugepages_nr_2M_pending \ if memory.vm_hugepages_nr_2M_pending is not None \ else memory.vm_hugepages_nr_2M @@ -648,6 +703,9 @@ class PlatformPuppet(base.BasePuppet): vm_2M_pages.append(vm_hugepages_nr_2M) vm_1G_pages.append(vm_hugepages_nr_1G) + platform_reserved_memory = "(%s)" % ' '.join(platform_nodes) + vswitch_reserved_memory = "(%s)" % ' '.join(vswitch_nodes) + nr_hugepages_2Ms = "(%s)" % ' '.join(hugepages_2Ms) nr_hugepages_1Gs = "(%s)" % ' '.join(hugepages_1Gs) @@ -658,6 +716,10 @@ class PlatformPuppet(base.BasePuppet): vm_1G = "\"%s\"" % ','.join([str(i) for i in vm_1G_pages]) config.update({ + 'platform::compute::params::compute_base_reserved': + platform_reserved_memory, + 'platform::compute::params::compute_vswitch_reserved': + vswitch_reserved_memory, 'platform::compute::hugepage::params::nr_hugepages_2M': nr_hugepages_2Ms, 'platform::compute::hugepage::params::nr_hugepages_1G':