Add error information for subcloud upgrades
This commit includes additional information for the orchestrator during the subcloud upgrade failure stage, specifically at the steps that run migration scripts. Test plan: - PASS: Execute the 'upgrade_controller_simplex' command, forcing a failure in different migration scripts. Verify that the output message contains sufficient execution information. Closes-Bug: 2038950 Signed-off-by: fperez <fabrizio.perez@windriver.com> Change-Id: I9971f08e763856938517e1ef2e7a76443432f734
This commit is contained in:
parent
e9422e8bd7
commit
975490c8b4
|
@ -87,7 +87,9 @@ def execute_migration_scripts(from_release, to_release, action,
|
||||||
for f in files:
|
for f in files:
|
||||||
migration_script = os.path.join(migration_script_dir, f)
|
migration_script = os.path.join(migration_script_dir, f)
|
||||||
try:
|
try:
|
||||||
LOG.info("Executing migration script %s" % migration_script)
|
# needed to flag each execution in case of error
|
||||||
|
start_script_exec = "Executing migration script"
|
||||||
|
LOG.info("%s %s" % (start_script_exec, migration_script))
|
||||||
# TODO(heitormatsui): remove py2 code when
|
# TODO(heitormatsui): remove py2 code when
|
||||||
# CentOS and zuul py2.7 are deprecated
|
# CentOS and zuul py2.7 are deprecated
|
||||||
if six.PY2:
|
if six.PY2:
|
||||||
|
@ -116,8 +118,11 @@ def execute_migration_scripts(from_release, to_release, action,
|
||||||
ret.returncode,
|
ret.returncode,
|
||||||
output_script)
|
output_script)
|
||||||
LOG.error(msg)
|
LOG.error(msg)
|
||||||
msg_temp = search_script_output(PLATFORM_LOG, f)
|
start_script_line = get_exec_start_line(
|
||||||
save_temp_file(msg, msg_temp)
|
start_script_exec, PLATFORM_LOG)
|
||||||
|
error_message = search_script_output(
|
||||||
|
start_script_line, PLATFORM_LOG, f)
|
||||||
|
save_temp_file(msg, error_message)
|
||||||
raise Exception(msg)
|
raise Exception(msg)
|
||||||
|
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
|
@ -132,23 +137,70 @@ def execute_migration_scripts(from_release, to_release, action,
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
def search_script_output(file_name, script):
|
def get_exec_start_line(start_script_exec, file_name):
|
||||||
|
""" Search the last ocurrence of the start of the script.
|
||||||
|
Get the line number and use it to find the last start
|
||||||
|
of script execution in logs.
|
||||||
|
|
||||||
|
Used to prevent reading an outdated error log.
|
||||||
|
"""
|
||||||
cmd = [
|
cmd = [
|
||||||
"awk",
|
"awk",
|
||||||
"/{script}/ {{last_match = $0}} "
|
'/{pattern_to_find}/ {{last_match = $0; start_line = NR}}'
|
||||||
"END {{if (last_match) print last_match}}".format(script=script),
|
'END {{if (last_match) print start_line, last_match}}'
|
||||||
|
.format(pattern_to_find=start_script_exec),
|
||||||
file_name
|
file_name
|
||||||
]
|
]
|
||||||
|
start_line = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
process = subprocess.Popen(cmd,
|
process = subprocess.Popen(cmd,
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
stderr=subprocess.PIPE)
|
stderr=subprocess.PIPE)
|
||||||
output, error = process.communicate()
|
output, error = process.communicate()
|
||||||
last_match = output.decode().strip()
|
last_match = output.decode().strip().splitlines()
|
||||||
|
start_line, last_match = last_match[0].split(' ', 1)
|
||||||
|
start_line = int(start_line)
|
||||||
except Exception:
|
except Exception:
|
||||||
LOG.error("Failed to exec cmd. \n %s" % error)
|
LOG.error("Failed to exec cmd. \n %s" % error)
|
||||||
return None
|
return None
|
||||||
return last_match
|
return start_line
|
||||||
|
|
||||||
|
|
||||||
|
def search_script_output(start_script_line, file_name, script):
|
||||||
|
"""Search error lines for this script.
|
||||||
|
|
||||||
|
Then, compare the line number and just add the
|
||||||
|
lines after the start of the last execution.
|
||||||
|
"""
|
||||||
|
cmd = [
|
||||||
|
"awk",
|
||||||
|
'/{script}/ && /error|ERROR/ {{print NR, $0}}'.format(script=script),
|
||||||
|
file_name
|
||||||
|
]
|
||||||
|
error_list = []
|
||||||
|
error_string = ""
|
||||||
|
|
||||||
|
try:
|
||||||
|
process = subprocess.Popen(cmd,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE)
|
||||||
|
output, error = process.communicate()
|
||||||
|
error_lines = output.decode().strip().splitlines()
|
||||||
|
# Compare the line numbers of each occurrence.
|
||||||
|
# If the line number is greater than 'start_script_line', then
|
||||||
|
# add this line to the output string
|
||||||
|
for i, current_line in enumerate(error_lines):
|
||||||
|
if i < (len(error_lines) - 1):
|
||||||
|
current_line, error_line = error_lines[i + 1].split(' ', 1)
|
||||||
|
current_line = int(current_line)
|
||||||
|
if current_line > start_script_line:
|
||||||
|
error_list.append(error_line)
|
||||||
|
error_string = '\n'.join(error_list)
|
||||||
|
except Exception:
|
||||||
|
LOG.error("Failed to exec cmd. \n %s" % error)
|
||||||
|
return None
|
||||||
|
return error_string
|
||||||
|
|
||||||
|
|
||||||
def save_temp_file(msg, error=None):
|
def save_temp_file(msg, error=None):
|
||||||
|
@ -157,8 +209,8 @@ def save_temp_file(msg, error=None):
|
||||||
|
|
||||||
MSG_FAILURE = '%s \n\n'\
|
MSG_FAILURE = '%s \n\n'\
|
||||||
'%s \n\n'\
|
'%s \n\n'\
|
||||||
'Check specific service log or search for' \
|
'Check specific service log or search for ' \
|
||||||
'this app in sysinv.log for details'
|
'this app in sysinv.log for details\n'
|
||||||
msg = MSG_FAILURE % (msg,
|
msg = MSG_FAILURE % (msg,
|
||||||
error)
|
error)
|
||||||
try:
|
try:
|
||||||
|
|
Loading…
Reference in New Issue