165 lines
5.4 KiB
Python
165 lines
5.4 KiB
Python
#!/usr/bin/python3
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
"""
|
|
Contains helper functions that will configure basic system settings.
|
|
"""
|
|
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from consts.timeout import HostTimeout
|
|
from utils import kpi, serial
|
|
from utils.install_log import LOG
|
|
from helper import host_helper
|
|
|
|
|
|
def update_platform_cpus(stream, hostname, cpu_num=5):
|
|
"""
|
|
Update platform CPU allocation.
|
|
"""
|
|
|
|
LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num, hostname)
|
|
cmd = "\nsource /etc/platform/openrc;" \
|
|
f" system host-cpu-modify {hostname} -f platform -p0 {cpu_num}"
|
|
serial.send_bytes(stream, cmd, prompt="keystone", timeout=300)
|
|
|
|
|
|
def set_dns(stream, dns_ip):
|
|
"""
|
|
Perform DNS configuration on the system.
|
|
"""
|
|
|
|
LOG.info("Configuring DNS to %s.", dns_ip)
|
|
cmd = f"source /etc/platform/openrc; system dns-modify nameservers={dns_ip}"
|
|
serial.send_bytes(stream, cmd, prompt="keystone")
|
|
|
|
|
|
def config_controller(stream, password):
|
|
"""
|
|
Configure controller-0 using optional arguments
|
|
"""
|
|
|
|
LOG.info("Executing the bootstrap ansible playbook")
|
|
cmd = "ansible-playbook /usr/share/ansible/stx-ansible/playbooks/bootstrap.yml"
|
|
serial.send_bytes(stream, cmd, expect_prompt=False)
|
|
host_helper.check_password(stream, password=password)
|
|
serial.expect_bytes(stream, "~$", timeout=HostTimeout.LAB_CONFIG)
|
|
|
|
cmd = "echo [$?]"
|
|
serial.send_bytes(stream, cmd, expect_prompt=False, log=False)
|
|
ret = serial.expect_bytes(stream, "[0]", timeout=HostTimeout.NORMAL_OP, log=False)
|
|
if ret != 0:
|
|
LOG.info("Configuration failed. Exiting installer.")
|
|
raise SystemExit("Configcontroller failed")
|
|
LOG.info("Successful bootstrap ansible playbook execution")
|
|
|
|
|
|
def fault_tolerant(scale=1):
|
|
"""
|
|
Provides the scale argument to the fault-tolerant decorator.
|
|
|
|
Args:
|
|
- scale: re-attempt delay vector scale factor
|
|
|
|
Returns:
|
|
- fault-tolerant decorator.
|
|
"""
|
|
|
|
def fault_tolerant_decorator(func):
|
|
"""
|
|
Decorator to run a command in a fault-tolerant fashion.
|
|
|
|
Args:
|
|
- func: The function to be decorated.
|
|
|
|
Returns:
|
|
- fault-tolerant wrapper
|
|
"""
|
|
|
|
def fault_tolerant_wrapper(*args, **kwargs):
|
|
|
|
"""
|
|
Runs a command in a fault-tolerant fashion.
|
|
|
|
The function provides a recovery mechanism with progressive re-attempt delays
|
|
The first attempt is the normal command execution. If the command fails, the first
|
|
re-attempt runs after 2s, and the re-attempt delay goes increasing until 10 min.
|
|
The last re-attempts, with longer delays are intended to help the user to
|
|
salvage the ongoing installation, if the system does not recover automatically.
|
|
|
|
Intentionally, the function does not provide a return code, due to the following
|
|
reason. To ensure system integrity, the function stops the program execution,
|
|
if it can not achieve a successful result, after a maximum number of retries.
|
|
|
|
Hence, any subsequent functions may safely rely on the system integrity.
|
|
|
|
Args:
|
|
- cmd: The command to be executed.
|
|
|
|
Returns: None
|
|
"""
|
|
|
|
delay = HostTimeout.REATTEMPT_DELAY
|
|
reattempt_delay = scale*delay
|
|
max_attempts = len(reattempt_delay)
|
|
attempt = 1
|
|
while True:
|
|
cmd = kwargs['cmd']
|
|
try:
|
|
return_code = func(*args, **kwargs)
|
|
assert return_code == 0
|
|
break
|
|
except AssertionError as exc:
|
|
if attempt < max_attempts:
|
|
LOG.warning(
|
|
"#### Failed command:\n$ %s [attempt: %s/%s]\n",
|
|
cmd, attempt, max_attempts
|
|
)
|
|
LOG.info(
|
|
"Trying again after %s ... ",
|
|
kpi.get_formated_time(reattempt_delay[attempt])
|
|
)
|
|
time.sleep(reattempt_delay[attempt])
|
|
attempt = attempt + 1
|
|
else:
|
|
LOG.error(
|
|
"#### Failed command:\n$ %s [attempt: %s/%s]\n",
|
|
cmd, attempt, max_attempts
|
|
)
|
|
raise TimeoutError from exc
|
|
except Exception as exc: # pylint: disable=broad-except
|
|
LOG.error(
|
|
"#### Failed command:\n$ %s\nError: %s",
|
|
cmd, repr(exc)
|
|
)
|
|
sys.exit(1)
|
|
|
|
return fault_tolerant_wrapper
|
|
|
|
return fault_tolerant_decorator
|
|
|
|
|
|
def exec_cmd(cmd):
|
|
|
|
"""
|
|
Execute a local command on the host machine in a fault-tolerant fashion.
|
|
Refer to the fault_tolerant decorator for more details.
|
|
"""
|
|
|
|
@fault_tolerant()
|
|
def exec_cmd_ft(*args, **kwargs): # pylint: disable=unused-argument
|
|
|
|
LOG.info("#### Executing command on the host machine:\n$ %s\n", cmd)
|
|
with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process:
|
|
for line in iter(process.stdout.readline, b''):
|
|
LOG.info("%s", line.decode("utf-8").strip())
|
|
process.wait()
|
|
return process.returncode
|
|
|
|
|
|
exec_cmd_ft(**{'cmd': cmd})
|