starlingx
/
update
Code Issues Proposed changes

Implement deploy host for major release deployment 

This commit adds the capability to run deploy host
for a major release deployment (release upgrade).

To achieve this, this commit essentially changes
some code that is already used by patching to allow:

1. Create a new remote pointing to the to_release feed ostree
2. Pull the to_release ostree commit to sysroot ostree
3. Deploy the to_release ostree commit

This commit also includes some additional steps for
sysinv/puppet integration with USM, and fixes minor
flake8 issues on the files that are being changed.

Test Plan
PASS: run "deploy host" for major release deployment
      successfully on AIO-SX
PASS: run "deploy host" for major release deployment
      successfully on AIO-DX
PASS: (regression) run "deploy host" successfully for a
      patch release

Story: 2010676
Task: 49787

Signed-off-by: Heitor Matsui <heitorvieira.matsui@windriver.com>
Change-Id: Ib8b08d1cd85dcad7d6fc858e2fae623b5900cffc
This commit is contained in:
Heitor Matsui 2024-03-28 10:05:12 -03:00
parent 979bd27d90
commit 2bcfb854e4
5 changed files with 157 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
software/software/ostree_utils.py
View File

@ -169,22 +169,35 @@ def reset_ostree_repo_head(commit, repo_path):
raise OSTreeCommandFail(msg)
def pull_ostree_from_remote():
def pull_ostree_from_remote(remote=None):
"""
Pull from remote ostree to sysroot ostree
"""
cmd = "ostree pull %s --depth=-1" % constants.OSTREE_REMOTE
cmd = "ostree pull %s --depth=-1"
ref_cmd = ""
if not remote:
ref = constants.OSTREE_REMOTE
else:
ref = "%s:%s" % (remote, constants.OSTREE_REF)
cmd += " --mirror"
ref_cmd = "ostree refs --create=%s %s" % (ref, constants.OSTREE_REF)
try:
subprocess.run(cmd, shell=True, check=True, capture_output=True)
subprocess.run(cmd % ref, shell=True, check=True, capture_output=True)
except subprocess.CalledProcessError as e:
msg = "Failed to pull from %s remote into sysroot ostree" % constants.OSTREE_REMOTE
msg = "Failed to pull from %s remote into sysroot ostree" % ref
info_msg = "OSTree Pull Error: return code: %s , Output: %s" \
% (e.returncode, e.stderr.decode("utf-8"))
LOG.info(info_msg)
raise OSTreeCommandFail(msg)
if ref_cmd:
try:
subprocess.run(ref_cmd, shell=True, check=True)
except subprocess.CalledProcessError:
msg = "Failed to create ref %s for remote %s" % (ref, remote)
def delete_ostree_repo_commit(commit, repo_path):
"""
@ -206,16 +219,19 @@ def delete_ostree_repo_commit(commit, repo_path):
raise OSTreeCommandFail(msg)
def create_deployment():
def create_deployment(ref=None):
"""
Create a new deployment while retaining the previous ones
"""
cmd = "ostree admin deploy %s --no-prune --retain" % constants.OSTREE_REF
if not ref:
ref = constants.OSTREE_REF
cmd = "ostree admin deploy %s --no-prune --retain" % ref
try:
subprocess.run(cmd, shell=True, check=True, capture_output=True)
except subprocess.CalledProcessError as e:
msg = "Failed to create an ostree deployment for sysroot ref %s." % constants.OSTREE_REF
msg = "Failed to create an ostree deployment for sysroot ref %s." % ref
info_msg = "OSTree Deployment Error: return code: %s , Output: %s" \
% (e.returncode, e.stderr.decode("utf-8"))
LOG.info(info_msg)
@ -507,3 +523,26 @@ def write_to_feed_ostree(patch_name, patch_sw_version):
% (vars(e))
LOG.info(info_msg)
raise OSTreeCommandFail(msg)
def add_ostree_remote(major_release, nodetype):
"""
Add a new ostree remote from a major release feed
:param major_release: major release corresponding to the new remote
:param nodetype: type of the node where the software agent is running
"""
rel_name = "rel-%s" % major_release
if nodetype == "controller":
feed_ostree_url = "file://%s/%s/ostree_repo/" % (
constants.FEED_OSTREE_BASE_DIR, rel_name)
else:
feed_ostree_url = "http://%s:8080/feed/%s/ostree_repo/" % (
constants.CONTROLLER_FLOATING_HOSTNAME, rel_name)
cmd = ["ostree", "remote", "add", "--no-gpg-verify",
"--if-not-exists", rel_name, feed_ostree_url, constants.OSTREE_REF]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError as e:
LOG.exception("Error adding %s ostree remote: %s" % (major_release, str(e)))
raise
return rel_name

68
software/software/software_agent.py
View File

@ -76,12 +76,12 @@ def pull_restart_scripts_from_controller():
# are not present, it should not raise any exception
try:
output = subprocess.check_output(["rsync",
"-acv",
"--delete",
"--exclude", "tmp",
"rsync://controller/repo/software-scripts/",
"%s/" % insvc_software_scripts],
stderr=subprocess.STDOUT)
"-acv",
"--delete",
"--exclude", "tmp",
"rsync://controller/repo/software-scripts/",
"%s/" % insvc_software_scripts],
stderr=subprocess.STDOUT)
LOG.info("Synced restart scripts from controller: %s", output)
except subprocess.CalledProcessError as e:
if "No such file or directory" in e.output.decode("utf-8"):
@ -117,6 +117,28 @@ def check_install_uuid():
return True
def copy_target_release_pxeboot_files(major_release):
"""
Copy pxeboot files from the target feed during
major release deployment. These files are copied
during the release upload, but only to the host
where it is uploaded, so this method is needed to
copy the files to other hosts.
:param major_release: target major release
"""
# copy to_release pxeboot files to /var/pxeboot/pxelinux.cfg.files
pxeboot_feed_dir = "/var/www/pages/feed/rel-%s/pxeboot/pxelinux.cfg.files/*" % major_release
pxeboot_dest_dir = "/var/pxeboot/pxelinux.cfg.files/"
cmd = "rsync -ac %s %s" % (pxeboot_feed_dir, pxeboot_dest_dir)
try:
subprocess.check_call(cmd, shell=True)
LOG.info("Copied %s pxeboot files successfully to %s." % (major_release, pxeboot_dest_dir))
except subprocess.CalledProcessError as e:
LOG.exception("Error copying pxeboot files from %s to %s: %s" % (
pxeboot_feed_dir, pxeboot_dest_dir, str(e)))
class PatchMessageSendLatestFeedCommit(messages.PatchMessage):
def __init__(self):
messages.PatchMessage.__init__(self, messages.PATCHMSG_SEND_LATEST_FEED_COMMIT)
@ -317,18 +339,21 @@ class PatchMessageAgentInstallReq(messages.PatchMessage):
def __init__(self):
messages.PatchMessage.__init__(self, messages.PATCHMSG_AGENT_INSTALL_REQ)
self.force = False
self.major_release = None
def decode(self, data):
messages.PatchMessage.decode(self, data)
if 'force' in data:
self.force = data['force']
if 'major_release' in data:
self.major_release = data['major_release']
def encode(self):
# Nothing to add to the HELLO_AGENT, so just call the super class
messages.PatchMessage.encode(self)
def handle(self, sock, addr):
LOG.info("Handling host install request, force=%s", self.force)
LOG.info("Handling host install request, force=%s, major_release=%s", self.force, self.major_release)
global pa
resp = PatchMessageAgentInstallResp()
@ -346,7 +371,7 @@ class PatchMessageAgentInstallReq(messages.PatchMessage):
resp.reject_reason = 'Node must be locked.'
resp.send(sock, addr)
return
resp.status = pa.handle_install()
resp.status = pa.handle_install(major_release=self.major_release)
resp.send(sock, addr)
def send(self, sock): # pylint: disable=unused-argument
@ -433,7 +458,7 @@ class PatchAgent(PatchService):
self.listener.bind(('', self.port))
self.listener.listen(2) # Allow two connections, for two controllers
def query(self):
def query(self, major_release=None):
"""Check current patch state """
if not check_install_uuid():
LOG.info("Failed install_uuid check. Skipping query")
@ -449,6 +474,13 @@ class PatchAgent(PatchService):
self.latest_sysroot_commit = active_sysroot_commit
self.last_repo_revision = active_sysroot_commit
if major_release:
upgrade_feed_commit = ostree_utils.get_feed_latest_commit(major_release)
LOG.info("Major release deployment for %s with commit %s" % (major_release,
upgrade_feed_commit))
self.changes = True
return True
# latest_feed_commit is sent from patch controller
# if unprovisioned (no mgmt ip) attempt to query it
if self.latest_feed_commit is None:
@ -471,7 +503,8 @@ class PatchAgent(PatchService):
def handle_install(self,
verbose_to_stdout=False,
disallow_insvc_patch=False,
delete_older_deployments=False):
delete_older_deployments=False,
major_release=None):
#
# The disallow_insvc_patch parameter is set when we're installing
# the patch during init. At that time, we don't want to deal with
@ -522,10 +555,19 @@ class PatchAgent(PatchService):
hello_ack = PatchMessageHelloAgentAck()
hello_ack.send(self.sock_out)
remote = None
ref = None
if major_release:
nodetype = utils.get_platform_conf("nodetype")
remote = ostree_utils.add_ostree_remote(major_release, nodetype)
ref = "%s:%s" % (remote, constants.OSTREE_REF)
LOG.info("OSTree remote added: %s" % remote)
copy_target_release_pxeboot_files(major_release)
# Build up the install set
if verbose_to_stdout:
print("Checking for software updates...")
self.query() # sets self.changes
self.query(major_release=major_release) # sets self.changes
changed = False
success = True
@ -537,7 +579,7 @@ class PatchAgent(PatchService):
# Pull changes from remote to the sysroot ostree
# The remote value is configured inside
# "/sysroot/ostree/repo/config" file
ostree_utils.pull_ostree_from_remote()
ostree_utils.pull_ostree_from_remote(remote=remote)
setflag(ostree_pull_completed_deployment_pending_file)
except OSTreeCommandFail:
LOG.exception("Failed to pull changes and create deployment"
@ -546,7 +588,7 @@ class PatchAgent(PatchService):
try:
# Create a new deployment once the changes are pulled
ostree_utils.create_deployment()
ostree_utils.create_deployment(ref=ref)
changed = True
clearflag(ostree_pull_completed_deployment_pending_file)

23
software/software/software_controller.py
View File

@ -62,6 +62,7 @@ from software.software_functions import PatchFile
from software.software_functions import package_dir
from software.software_functions import repo_dir
from software.software_functions import root_scripts_dir
from software.software_functions import set_host_target_load
from software.software_functions import SW_VERSION
from software.software_functions import LOG
from software.software_functions import audit_log_info
@ -524,11 +525,13 @@ class PatchMessageAgentInstallReq(messages.PatchMessage):
messages.PatchMessage.__init__(self, messages.PATCHMSG_AGENT_INSTALL_REQ)
self.ip = None
self.force = False
self.major_release = None
def encode(self):
global sc
messages.PatchMessage.encode(self)
self.message['force'] = self.force
self.message['major_release'] = self.major_release
def handle(self, sock, addr):
LOG.error("Should not get here")
@ -2373,8 +2376,8 @@ class PatchController(PatchService):
return ret
elif not ret["system_healthy"]:
ret["info"] = "The following issues have been detected, which prevent " \
"deploying %s\n" % deployment + ret["info"] + \
"Please fix above issues then retry the deploy.\n"
"deploying %s\n" % deployment + ret["info"] + \
"Please fix above issues then retry the deploy.\n"
return ret
if self._deploy_upgrade_start(to_release):
@ -2805,6 +2808,21 @@ class PatchController(PatchService):
force = True
self.copy_restart_scripts()
# Check if there is a major release deployment in progress
# and set agent request parameters accordingly
major_release = None
upgrade_in_progress = self.get_software_upgrade()
if upgrade_in_progress:
major_release = upgrade_in_progress["to_release"]
force = False
async_req = False
msg = "Running major release deployment, major_release=%s, force=%s, async_req=%s" % (
major_release, force, async_req)
msg_info += msg + "\n"
LOG.info(msg)
set_host_target_load(host_ip, major_release)
# TODO(heitormatsui) update host deploy status
self.hosts[ip].install_pending = True
self.hosts[ip].install_status = False
self.hosts[ip].install_reject_reason = None
@ -2813,6 +2831,7 @@ class PatchController(PatchService):
installreq = PatchMessageAgentInstallReq()
installreq.ip = ip
installreq.force = force
installreq.major_release = major_release
installreq.encode()
self.socket_lock.acquire()
installreq.send(self.sock_out)

25
software/software/software_functions.py
View File

@ -1323,3 +1323,28 @@ def is_deployment_in_progress(release_metadata):
:return: bool true if in progress, false otherwise
"""
return any(release['state'] == constants.DEPLOYING for release in release_metadata.values())
def set_host_target_load(hostname, major_release):
"""
Set target_load on the sysinv db for a host during deploy
host for major release deployment. This action is needed
so that sysinv behaves correctly when the host is unlocked
and after it reboots running the new software load.
:param hostname: host being deployed
:param major_release: target major release
TODO(heitormatsui): delete this function once sysinv upgrade tables are deprecated
"""
load_query = "select id from loads where software_version = '%s'" % major_release
host_query = "select id from i_host where hostname = '%s'" % hostname
update_query = ("update host_upgrade set software_load = (%s), target_load = (%s) "
"where forihostid = (%s)") % (load_query, load_query, host_query)
cmd = "sudo -u postgres psql -d sysinv -c \"%s\"" % update_query
try:
subprocess.check_call(cmd, shell=True)
LOG.info("Host %s target_load set to %s" % (hostname, major_release))
except subprocess.CalledProcessError as e:
LOG.exception("Error setting target_load to %s for %s: %s" % (
major_release, hostname, str(e)))
raise

18
software/software/utils.py
View File

@ -4,6 +4,7 @@ Copyright (c) 2023-2024 Wind River Systems, Inc.
SPDX-License-Identifier: Apache-2.0
"""
import configparser
import hashlib
from pecan import hooks
import json
@ -446,16 +447,17 @@ def get_platform_conf(key):
"""
Get the value of given key in platform.conf
:param key: key to get
:return: value
:return: value corresponding to key
"""
default_section = "DEFAULT"
value = None
with open(PLATFORM_CONF_FILE) as fp:
lines = fp.readlines()
for line in lines:
if line.find(key) != -1:
value = line.split('=')[1]
value = value.replace('\n', '')
break
config = ("[%s]\n" % default_section) + fp.read()
cp = configparser.ConfigParser()
try:
cp.read_string(config)
value = cp[default_section][key]
except Exception:
LOG.error("Cannot get '%s' from platform.conf file." % key)
return value