399 lines
11 KiB
Puppet
399 lines
11 KiB
Puppet
class platform::compute::params (
|
|
$worker_cpu_list = '',
|
|
$platform_cpu_list = '',
|
|
$reserved_vswitch_cores = '',
|
|
$reserved_platform_cores = '',
|
|
$worker_base_reserved = '',
|
|
$compute_vswitch_reserved = '',
|
|
) { }
|
|
|
|
class platform::compute::config
|
|
inherits ::platform::compute::params {
|
|
include ::platform::collectd::restart
|
|
file { '/etc/platform/worker_reserved.conf':
|
|
ensure => 'present',
|
|
replace => true,
|
|
content => template('platform/worker_reserved.conf.erb')
|
|
}
|
|
-> Exec['collectd-restart']
|
|
|
|
if $::platform::params::system_type != 'All-in-one' {
|
|
file { '/etc/systemd/system.conf.d/platform-cpuaffinity.conf':
|
|
ensure => 'present',
|
|
replace => true,
|
|
content => template('platform/systemd-system-cpuaffinity.conf.erb')
|
|
}
|
|
}
|
|
}
|
|
|
|
class platform::compute::config::runtime {
|
|
include ::platform::compute::config
|
|
}
|
|
|
|
class platform::compute::grub::params (
|
|
$n_cpus = '',
|
|
$cpu_options = '',
|
|
$m_hugepages = 'hugepagesz=2M hugepages=0',
|
|
$g_hugepages = undef,
|
|
$default_pgsz = 'default_hugepagesz=2M',
|
|
$keys = [
|
|
'kvm-intel.eptad',
|
|
'default_hugepagesz',
|
|
'hugepagesz',
|
|
'hugepages',
|
|
'isolcpus',
|
|
'nohz_full',
|
|
'rcu_nocbs',
|
|
'kthread_cpus',
|
|
'irqaffinity',
|
|
],
|
|
) {
|
|
|
|
if $::is_broadwell_processor {
|
|
$eptad = 'kvm-intel.eptad=0'
|
|
} else {
|
|
$eptad = ''
|
|
}
|
|
|
|
if $::is_gb_page_supported and $::platform::params::vswitch_type != 'none' {
|
|
if $g_hugepages != undef {
|
|
$gb_hugepages = $g_hugepages
|
|
} else {
|
|
$gb_hugepages = "hugepagesz=1G hugepages=${::number_of_numa_nodes}"
|
|
}
|
|
} else {
|
|
$gb_hugepages = ''
|
|
}
|
|
|
|
$grub_updates = strip("${eptad} ${$gb_hugepages} ${m_hugepages} ${default_pgsz} ${cpu_options}")
|
|
}
|
|
|
|
class platform::compute::grub::update
|
|
inherits ::platform::compute::grub::params {
|
|
|
|
notice('Updating grub configuration')
|
|
|
|
$to_be_removed = join($keys, ' ')
|
|
exec { 'Remove the cpu arguments':
|
|
command => "grubby --update-kernel=ALL --remove-args='${to_be_removed}'",
|
|
}
|
|
-> exec { 'Add the cpu arguments':
|
|
command => "grubby --update-kernel=ALL --args='${grub_updates}'",
|
|
}
|
|
}
|
|
|
|
class platform::compute::grub::recovery {
|
|
|
|
notice('Update Grub and Reboot')
|
|
|
|
class {'platform::compute::grub::update': } -> Exec['reboot-recovery']
|
|
|
|
exec { 'reboot-recovery':
|
|
command => 'reboot',
|
|
}
|
|
}
|
|
|
|
class platform::compute::grub::audit
|
|
inherits ::platform::compute::grub::params {
|
|
|
|
if ! str2bool($::is_initial_config_primary) {
|
|
|
|
notice('Audit CPU and Grub Configuration')
|
|
|
|
$expected_n_cpus = Integer($::number_of_logical_cpus)
|
|
$n_cpus_ok = ($n_cpus == $expected_n_cpus)
|
|
|
|
$cmd_ok = check_grub_config($grub_updates)
|
|
|
|
if $cmd_ok and $n_cpus_ok {
|
|
$ensure = present
|
|
notice('CPU and Boot Argument audit passed.')
|
|
} else {
|
|
$ensure = absent
|
|
if !$cmd_ok {
|
|
notice('Kernel Boot Argument Mismatch')
|
|
include ::platform::compute::grub::recovery
|
|
} else {
|
|
notice("Mismatched CPUs: Found=${n_cpus}, Expected=${expected_n_cpus}")
|
|
}
|
|
}
|
|
|
|
file { '/var/run/worker_goenabled':
|
|
ensure => $ensure,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0644',
|
|
}
|
|
}
|
|
}
|
|
|
|
class platform::compute::grub::runtime {
|
|
include ::platform::compute::grub::update
|
|
}
|
|
|
|
# Mounts virtual hugetlbfs filesystems for each supported page size
|
|
class platform::compute::hugetlbf {
|
|
|
|
if str2bool($::is_hugetlbfs_enabled) {
|
|
|
|
$fs_list = generate('/bin/bash', '-c', 'ls -1d /sys/kernel/mm/hugepages/hugepages-*')
|
|
$array = split($fs_list, '\n')
|
|
$array.each | String $val | {
|
|
$page_name = generate('/bin/bash', '-c', "basename ${val}")
|
|
$page_size = strip(regsubst($page_name, 'hugepages-', ''))
|
|
$hugemnt ="/mnt/huge-${page_size}"
|
|
$options = "pagesize=${page_size}"
|
|
|
|
# TODO: Once all the code is switched over to use the /dev
|
|
# mount point we can get rid of this mount point.
|
|
notice("Mounting hugetlbfs at: ${hugemnt}")
|
|
exec { "create ${hugemnt}":
|
|
command => "mkdir -p ${hugemnt}",
|
|
onlyif => "test ! -d ${hugemnt}",
|
|
}
|
|
-> mount { $hugemnt:
|
|
ensure => 'mounted',
|
|
device => 'none',
|
|
fstype => 'hugetlbfs',
|
|
name => $hugemnt,
|
|
options => $options,
|
|
atboot => 'yes',
|
|
remounts => true,
|
|
}
|
|
|
|
# The libvirt helm chart expects hugepages to be mounted
|
|
# under /dev so let's do that.
|
|
$hugemnt2 ="/dev/huge-${page_size}"
|
|
notice("Mounting hugetlbfs at: ${hugemnt2}")
|
|
file { $hugemnt2:
|
|
ensure => 'directory',
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0755',
|
|
}
|
|
-> mount { $hugemnt2:
|
|
ensure => 'mounted',
|
|
device => 'none',
|
|
fstype => 'hugetlbfs',
|
|
name => $hugemnt2,
|
|
options => $options,
|
|
atboot => 'yes',
|
|
remounts => true,
|
|
}
|
|
}
|
|
|
|
# The libvirt helm chart also assumes that the default hugepage size
|
|
# will be mounted at /dev/hugepages so let's make that happen too.
|
|
# Once we upstream a fix to the helm chart to automatically determine
|
|
# the mountpoint then we can remove this.
|
|
$page_size = '2M'
|
|
$hugemnt ='/dev/hugepages'
|
|
$options = "pagesize=${page_size}"
|
|
|
|
notice("Mounting hugetlbfs at: ${hugemnt}")
|
|
exec { "create ${hugemnt}":
|
|
command => "mkdir -p ${hugemnt}",
|
|
onlyif => "test ! -d ${hugemnt}",
|
|
}
|
|
-> mount { $hugemnt:
|
|
ensure => 'mounted',
|
|
device => 'none',
|
|
fstype => 'hugetlbfs',
|
|
name => $hugemnt,
|
|
options => $options,
|
|
atboot => 'yes',
|
|
remounts => true,
|
|
}
|
|
}
|
|
}
|
|
|
|
# lint:ignore:variable_is_lowercase
|
|
class platform::compute::hugepage::params (
|
|
$nr_hugepages_2M = undef,
|
|
$nr_hugepages_1G = undef,
|
|
$vswitch_2M_pages = '',
|
|
$vswitch_1G_pages = '',
|
|
$vm_4K_pages = '',
|
|
$vm_2M_pages = '',
|
|
$vm_1G_pages = '',
|
|
) {}
|
|
|
|
|
|
define allocate_pages (
|
|
$path,
|
|
$page_count,
|
|
) {
|
|
exec { "Allocate ${page_count} ${path}":
|
|
command => "echo ${page_count} > ${path}",
|
|
onlyif => "test -f ${path}",
|
|
}
|
|
}
|
|
|
|
# Allocates HugeTLB memory according to the attributes specified in the
|
|
# nr_hugepages_2M and nr_hugepages_1G
|
|
class platform::compute::allocate
|
|
inherits ::platform::compute::hugepage::params {
|
|
|
|
# determine the node file system
|
|
if str2bool($::is_per_numa_supported) {
|
|
$nodefs = '/sys/devices/system/node'
|
|
} else {
|
|
$nodefs = '/sys/kernel/mm'
|
|
}
|
|
|
|
if $nr_hugepages_2M != undef {
|
|
$nr_hugepages_2M_array = regsubst($nr_hugepages_2M, '[\(\)\"]', '', 'G').split(' ')
|
|
$nr_hugepages_2M_array.each | String $val | {
|
|
$per_node_2M = $val.split(':')
|
|
if size($per_node_2M)== 3 {
|
|
$node = $per_node_2M[0]
|
|
$page_size = $per_node_2M[1]
|
|
allocate_pages { "Start ${node} ${page_size}":
|
|
path => "${nodefs}/${node}/hugepages/hugepages-${page_size}/nr_hugepages",
|
|
page_count => $per_node_2M[2],
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if $nr_hugepages_1G != undef {
|
|
$nr_hugepages_1G_array = regsubst($nr_hugepages_1G , '[\(\)\"]', '', 'G').split(' ')
|
|
$nr_hugepages_1G_array.each | String $val | {
|
|
$per_node_1G = $val.split(':')
|
|
if size($per_node_1G)== 3 {
|
|
$node = $per_node_1G[0]
|
|
$page_size = $per_node_1G[1]
|
|
allocate_pages { "Start ${node} ${page_size}":
|
|
path => "${nodefs}/${node}/hugepages/hugepages-${page_size}/nr_hugepages",
|
|
page_count => $per_node_1G[2],
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
# lint:endignore:variable_is_lowercase
|
|
|
|
# Mount resctrl to allow Cache Allocation Technology per VM
|
|
class platform::compute::resctrl {
|
|
|
|
if str2bool($::is_resctrl_supported) {
|
|
mount { '/sys/fs/resctrl':
|
|
ensure => 'mounted',
|
|
device => 'resctrl',
|
|
fstype => 'resctrl',
|
|
name => '/sys/fs/resctrl',
|
|
atboot => 'yes',
|
|
remounts => true,
|
|
}
|
|
}
|
|
}
|
|
|
|
# Set Power Management QoS resume latency constraints for CPUs.
|
|
# The PM QoS resume latency limit is set to shallow C-state for vswitch CPUs.
|
|
# All other CPUs are allowed to go to the deepest C-state available.
|
|
class platform::compute::pmqos (
|
|
$low_wakeup_cpus = '',
|
|
$hight_wakeup_cpus = '',
|
|
) {
|
|
|
|
if str2bool($::is_worker_subfunction) and str2bool($::is_lowlatency_subfunction) {
|
|
|
|
$script = '/usr/bin/set-cpu-wakeup-latency.sh'
|
|
|
|
if $low_wakeup_cpus != '""' {
|
|
# Set low wakeup latency (shallow C-state) for vswitch CPUs using PM QoS interface
|
|
exec { 'low-wakeup-latency':
|
|
command => "${script} low ${low_wakeup_cpus}",
|
|
onlyif => "test -f ${script}",
|
|
logoutput => true,
|
|
}
|
|
}
|
|
|
|
if $hight_wakeup_cpus != '""' {
|
|
#Set high wakeup latency (deep C-state) for non-vswitch CPUs using PM QoS interface
|
|
exec { 'high-wakeup-latency':
|
|
command => "${script} high ${hight_wakeup_cpus}",
|
|
onlyif => "test -f ${script}",
|
|
logoutput => true,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Set systemd machine.slice cgroup cpuset to be used with VMs,
|
|
# and configure this cpuset to span all logical cpus and numa nodes.
|
|
# NOTES:
|
|
# - The parent directory cpuset spans all online cpus and numa nodes.
|
|
# - Setting the machine.slice cpuset prevents this from inheriting
|
|
# kubernetes libvirt pod's cpuset, since machine.slice cgroup will be
|
|
# created when a VM is launched if it does not already exist.
|
|
# - systemd automatically mounts cgroups and controllers, so don't need
|
|
# to do that here.
|
|
class platform::compute::machine {
|
|
$parent_dir = '/sys/fs/cgroup/cpuset'
|
|
$parent_mems = "${parent_dir}/cpuset.mems"
|
|
$parent_cpus = "${parent_dir}/cpuset.cpus"
|
|
$machine_dir = "${parent_dir}/machine.slice"
|
|
$machine_mems = "${machine_dir}/cpuset.mems"
|
|
$machine_cpus = "${machine_dir}/cpuset.cpus"
|
|
notice("Create ${machine_dir}")
|
|
file { $machine_dir :
|
|
ensure => directory,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0700',
|
|
}
|
|
-> exec { "Create ${machine_mems}" :
|
|
command => "/bin/cat ${parent_mems} > ${machine_mems}",
|
|
}
|
|
-> exec { "Create ${machine_cpus}" :
|
|
command => "/bin/cat ${parent_cpus} > ${machine_cpus}",
|
|
}
|
|
}
|
|
|
|
class platform::compute::kvm_timer_advance(
|
|
$enabled = False,
|
|
$vcpu_pin_set = undef
|
|
) {
|
|
if $enabled {
|
|
# include the declaration of the kubelet service
|
|
include ::platform::kubernetes::worker
|
|
|
|
file { '/etc/kvm-timer-advance/kvm-timer-advance.conf':
|
|
ensure => 'present',
|
|
replace => true,
|
|
content => template('platform/kvm_timer_advance.conf.erb')
|
|
}
|
|
-> service { 'kvm_timer_advance_setup':
|
|
ensure => 'running',
|
|
enable => true,
|
|
before => Service['kubelet'],
|
|
}
|
|
# A separate enable is required since we have modified the service resource
|
|
# to never enable/disable services in puppet.
|
|
-> exec { 'Enable kvm_timer_advance_setup':
|
|
command => '/usr/bin/systemctl enable kvm_timer_advance_setup.service',
|
|
}
|
|
} else {
|
|
# A disable is required since we have modified the service resource
|
|
# to never enable/disable services in puppet and stop has no effect.
|
|
exec { 'Disable kvm_timer_advance_setup':
|
|
command => '/usr/bin/systemctl disable kvm_timer_advance_setup.service',
|
|
}
|
|
}
|
|
}
|
|
|
|
class platform::compute {
|
|
|
|
Class[$name] -> Class['::platform::vswitch']
|
|
|
|
require ::platform::compute::grub::audit
|
|
require ::platform::compute::hugetlbf
|
|
require ::platform::compute::allocate
|
|
require ::platform::compute::pmqos
|
|
require ::platform::compute::resctrl
|
|
require ::platform::compute::machine
|
|
require ::platform::compute::config
|
|
require ::platform::compute::kvm_timer_advance
|
|
}
|