diff --git a/software/software/ostree_utils.py b/software/software/ostree_utils.py index 52c8a282..c5e713e1 100644 --- a/software/software/ostree_utils.py +++ b/software/software/ostree_utils.py @@ -169,22 +169,35 @@ def reset_ostree_repo_head(commit, repo_path): raise OSTreeCommandFail(msg) -def pull_ostree_from_remote(): +def pull_ostree_from_remote(remote=None): """ Pull from remote ostree to sysroot ostree """ - cmd = "ostree pull %s --depth=-1" % constants.OSTREE_REMOTE + cmd = "ostree pull %s --depth=-1" + ref_cmd = "" + if not remote: + ref = constants.OSTREE_REMOTE + else: + ref = "%s:%s" % (remote, constants.OSTREE_REF) + cmd += " --mirror" + ref_cmd = "ostree refs --create=%s %s" % (ref, constants.OSTREE_REF) try: - subprocess.run(cmd, shell=True, check=True, capture_output=True) + subprocess.run(cmd % ref, shell=True, check=True, capture_output=True) except subprocess.CalledProcessError as e: - msg = "Failed to pull from %s remote into sysroot ostree" % constants.OSTREE_REMOTE + msg = "Failed to pull from %s remote into sysroot ostree" % ref info_msg = "OSTree Pull Error: return code: %s , Output: %s" \ % (e.returncode, e.stderr.decode("utf-8")) LOG.info(info_msg) raise OSTreeCommandFail(msg) + if ref_cmd: + try: + subprocess.run(ref_cmd, shell=True, check=True) + except subprocess.CalledProcessError: + msg = "Failed to create ref %s for remote %s" % (ref, remote) + def delete_ostree_repo_commit(commit, repo_path): """ @@ -206,16 +219,19 @@ def delete_ostree_repo_commit(commit, repo_path): raise OSTreeCommandFail(msg) -def create_deployment(): +def create_deployment(ref=None): """ Create a new deployment while retaining the previous ones """ - cmd = "ostree admin deploy %s --no-prune --retain" % constants.OSTREE_REF + if not ref: + ref = constants.OSTREE_REF + cmd = "ostree admin deploy %s --no-prune --retain" % ref + try: subprocess.run(cmd, shell=True, check=True, capture_output=True) except subprocess.CalledProcessError as e: - msg = "Failed to create an ostree deployment for sysroot ref %s." % constants.OSTREE_REF + msg = "Failed to create an ostree deployment for sysroot ref %s." % ref info_msg = "OSTree Deployment Error: return code: %s , Output: %s" \ % (e.returncode, e.stderr.decode("utf-8")) LOG.info(info_msg) @@ -507,3 +523,26 @@ def write_to_feed_ostree(patch_name, patch_sw_version): % (vars(e)) LOG.info(info_msg) raise OSTreeCommandFail(msg) + + +def add_ostree_remote(major_release, nodetype): + """ + Add a new ostree remote from a major release feed + :param major_release: major release corresponding to the new remote + :param nodetype: type of the node where the software agent is running + """ + rel_name = "rel-%s" % major_release + if nodetype == "controller": + feed_ostree_url = "file://%s/%s/ostree_repo/" % ( + constants.FEED_OSTREE_BASE_DIR, rel_name) + else: + feed_ostree_url = "http://%s:8080/feed/%s/ostree_repo/" % ( + constants.CONTROLLER_FLOATING_HOSTNAME, rel_name) + cmd = ["ostree", "remote", "add", "--no-gpg-verify", + "--if-not-exists", rel_name, feed_ostree_url, constants.OSTREE_REF] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + LOG.exception("Error adding %s ostree remote: %s" % (major_release, str(e))) + raise + return rel_name diff --git a/software/software/software_agent.py b/software/software/software_agent.py index bcac170a..46549a4e 100644 --- a/software/software/software_agent.py +++ b/software/software/software_agent.py @@ -76,12 +76,12 @@ def pull_restart_scripts_from_controller(): # are not present, it should not raise any exception try: output = subprocess.check_output(["rsync", - "-acv", - "--delete", - "--exclude", "tmp", - "rsync://controller/repo/software-scripts/", - "%s/" % insvc_software_scripts], - stderr=subprocess.STDOUT) + "-acv", + "--delete", + "--exclude", "tmp", + "rsync://controller/repo/software-scripts/", + "%s/" % insvc_software_scripts], + stderr=subprocess.STDOUT) LOG.info("Synced restart scripts from controller: %s", output) except subprocess.CalledProcessError as e: if "No such file or directory" in e.output.decode("utf-8"): @@ -117,6 +117,28 @@ def check_install_uuid(): return True +def copy_target_release_pxeboot_files(major_release): + """ + Copy pxeboot files from the target feed during + major release deployment. These files are copied + during the release upload, but only to the host + where it is uploaded, so this method is needed to + copy the files to other hosts. + + :param major_release: target major release + """ + # copy to_release pxeboot files to /var/pxeboot/pxelinux.cfg.files + pxeboot_feed_dir = "/var/www/pages/feed/rel-%s/pxeboot/pxelinux.cfg.files/*" % major_release + pxeboot_dest_dir = "/var/pxeboot/pxelinux.cfg.files/" + cmd = "rsync -ac %s %s" % (pxeboot_feed_dir, pxeboot_dest_dir) + try: + subprocess.check_call(cmd, shell=True) + LOG.info("Copied %s pxeboot files successfully to %s." % (major_release, pxeboot_dest_dir)) + except subprocess.CalledProcessError as e: + LOG.exception("Error copying pxeboot files from %s to %s: %s" % ( + pxeboot_feed_dir, pxeboot_dest_dir, str(e))) + + class PatchMessageSendLatestFeedCommit(messages.PatchMessage): def __init__(self): messages.PatchMessage.__init__(self, messages.PATCHMSG_SEND_LATEST_FEED_COMMIT) @@ -317,18 +339,21 @@ class PatchMessageAgentInstallReq(messages.PatchMessage): def __init__(self): messages.PatchMessage.__init__(self, messages.PATCHMSG_AGENT_INSTALL_REQ) self.force = False + self.major_release = None def decode(self, data): messages.PatchMessage.decode(self, data) if 'force' in data: self.force = data['force'] + if 'major_release' in data: + self.major_release = data['major_release'] def encode(self): # Nothing to add to the HELLO_AGENT, so just call the super class messages.PatchMessage.encode(self) def handle(self, sock, addr): - LOG.info("Handling host install request, force=%s", self.force) + LOG.info("Handling host install request, force=%s, major_release=%s", self.force, self.major_release) global pa resp = PatchMessageAgentInstallResp() @@ -346,7 +371,7 @@ class PatchMessageAgentInstallReq(messages.PatchMessage): resp.reject_reason = 'Node must be locked.' resp.send(sock, addr) return - resp.status = pa.handle_install() + resp.status = pa.handle_install(major_release=self.major_release) resp.send(sock, addr) def send(self, sock): # pylint: disable=unused-argument @@ -433,7 +458,7 @@ class PatchAgent(PatchService): self.listener.bind(('', self.port)) self.listener.listen(2) # Allow two connections, for two controllers - def query(self): + def query(self, major_release=None): """Check current patch state """ if not check_install_uuid(): LOG.info("Failed install_uuid check. Skipping query") @@ -449,6 +474,13 @@ class PatchAgent(PatchService): self.latest_sysroot_commit = active_sysroot_commit self.last_repo_revision = active_sysroot_commit + if major_release: + upgrade_feed_commit = ostree_utils.get_feed_latest_commit(major_release) + LOG.info("Major release deployment for %s with commit %s" % (major_release, + upgrade_feed_commit)) + self.changes = True + return True + # latest_feed_commit is sent from patch controller # if unprovisioned (no mgmt ip) attempt to query it if self.latest_feed_commit is None: @@ -471,7 +503,8 @@ class PatchAgent(PatchService): def handle_install(self, verbose_to_stdout=False, disallow_insvc_patch=False, - delete_older_deployments=False): + delete_older_deployments=False, + major_release=None): # # The disallow_insvc_patch parameter is set when we're installing # the patch during init. At that time, we don't want to deal with @@ -522,10 +555,19 @@ class PatchAgent(PatchService): hello_ack = PatchMessageHelloAgentAck() hello_ack.send(self.sock_out) + remote = None + ref = None + if major_release: + nodetype = utils.get_platform_conf("nodetype") + remote = ostree_utils.add_ostree_remote(major_release, nodetype) + ref = "%s:%s" % (remote, constants.OSTREE_REF) + LOG.info("OSTree remote added: %s" % remote) + copy_target_release_pxeboot_files(major_release) + # Build up the install set if verbose_to_stdout: print("Checking for software updates...") - self.query() # sets self.changes + self.query(major_release=major_release) # sets self.changes changed = False success = True @@ -537,7 +579,7 @@ class PatchAgent(PatchService): # Pull changes from remote to the sysroot ostree # The remote value is configured inside # "/sysroot/ostree/repo/config" file - ostree_utils.pull_ostree_from_remote() + ostree_utils.pull_ostree_from_remote(remote=remote) setflag(ostree_pull_completed_deployment_pending_file) except OSTreeCommandFail: LOG.exception("Failed to pull changes and create deployment" @@ -546,7 +588,7 @@ class PatchAgent(PatchService): try: # Create a new deployment once the changes are pulled - ostree_utils.create_deployment() + ostree_utils.create_deployment(ref=ref) changed = True clearflag(ostree_pull_completed_deployment_pending_file) diff --git a/software/software/software_controller.py b/software/software/software_controller.py index 7056e7e7..6b8cc8cd 100644 --- a/software/software/software_controller.py +++ b/software/software/software_controller.py @@ -62,6 +62,7 @@ from software.software_functions import PatchFile from software.software_functions import package_dir from software.software_functions import repo_dir from software.software_functions import root_scripts_dir +from software.software_functions import set_host_target_load from software.software_functions import SW_VERSION from software.software_functions import LOG from software.software_functions import audit_log_info @@ -524,11 +525,13 @@ class PatchMessageAgentInstallReq(messages.PatchMessage): messages.PatchMessage.__init__(self, messages.PATCHMSG_AGENT_INSTALL_REQ) self.ip = None self.force = False + self.major_release = None def encode(self): global sc messages.PatchMessage.encode(self) self.message['force'] = self.force + self.message['major_release'] = self.major_release def handle(self, sock, addr): LOG.error("Should not get here") @@ -2373,8 +2376,8 @@ class PatchController(PatchService): return ret elif not ret["system_healthy"]: ret["info"] = "The following issues have been detected, which prevent " \ - "deploying %s\n" % deployment + ret["info"] + \ - "Please fix above issues then retry the deploy.\n" + "deploying %s\n" % deployment + ret["info"] + \ + "Please fix above issues then retry the deploy.\n" return ret if self._deploy_upgrade_start(to_release): @@ -2805,6 +2808,21 @@ class PatchController(PatchService): force = True self.copy_restart_scripts() + # Check if there is a major release deployment in progress + # and set agent request parameters accordingly + major_release = None + upgrade_in_progress = self.get_software_upgrade() + if upgrade_in_progress: + major_release = upgrade_in_progress["to_release"] + force = False + async_req = False + msg = "Running major release deployment, major_release=%s, force=%s, async_req=%s" % ( + major_release, force, async_req) + msg_info += msg + "\n" + LOG.info(msg) + set_host_target_load(host_ip, major_release) + # TODO(heitormatsui) update host deploy status + self.hosts[ip].install_pending = True self.hosts[ip].install_status = False self.hosts[ip].install_reject_reason = None @@ -2813,6 +2831,7 @@ class PatchController(PatchService): installreq = PatchMessageAgentInstallReq() installreq.ip = ip installreq.force = force + installreq.major_release = major_release installreq.encode() self.socket_lock.acquire() installreq.send(self.sock_out) diff --git a/software/software/software_functions.py b/software/software/software_functions.py index 0bff6986..e26adc6e 100644 --- a/software/software/software_functions.py +++ b/software/software/software_functions.py @@ -1323,3 +1323,28 @@ def is_deployment_in_progress(release_metadata): :return: bool true if in progress, false otherwise """ return any(release['state'] == constants.DEPLOYING for release in release_metadata.values()) + + +def set_host_target_load(hostname, major_release): + """ + Set target_load on the sysinv db for a host during deploy + host for major release deployment. This action is needed + so that sysinv behaves correctly when the host is unlocked + and after it reboots running the new software load. + + :param hostname: host being deployed + :param major_release: target major release + TODO(heitormatsui): delete this function once sysinv upgrade tables are deprecated + """ + load_query = "select id from loads where software_version = '%s'" % major_release + host_query = "select id from i_host where hostname = '%s'" % hostname + update_query = ("update host_upgrade set software_load = (%s), target_load = (%s) " + "where forihostid = (%s)") % (load_query, load_query, host_query) + cmd = "sudo -u postgres psql -d sysinv -c \"%s\"" % update_query + try: + subprocess.check_call(cmd, shell=True) + LOG.info("Host %s target_load set to %s" % (hostname, major_release)) + except subprocess.CalledProcessError as e: + LOG.exception("Error setting target_load to %s for %s: %s" % ( + major_release, hostname, str(e))) + raise diff --git a/software/software/utils.py b/software/software/utils.py index 618104fc..cd1b431c 100644 --- a/software/software/utils.py +++ b/software/software/utils.py @@ -4,6 +4,7 @@ Copyright (c) 2023-2024 Wind River Systems, Inc. SPDX-License-Identifier: Apache-2.0 """ +import configparser import hashlib from pecan import hooks import json @@ -446,16 +447,17 @@ def get_platform_conf(key): """ Get the value of given key in platform.conf :param key: key to get - :return: value + :return: value corresponding to key """ + default_section = "DEFAULT" value = None with open(PLATFORM_CONF_FILE) as fp: - lines = fp.readlines() - for line in lines: - if line.find(key) != -1: - value = line.split('=')[1] - value = value.replace('\n', '') - break - + config = ("[%s]\n" % default_section) + fp.read() + cp = configparser.ConfigParser() + try: + cp.read_string(config) + value = cp[default_section][key] + except Exception: + LOG.error("Cannot get '%s' from platform.conf file." % key) return value