Merge "Debian: In-service patch implementation"
This commit is contained in:
commit
def5e22b27
|
@ -46,6 +46,8 @@ OSTREE_REF = "starlingx"
|
|||
OSTREE_REMOTE = "debian"
|
||||
FEED_OSTREE_BASE_DIR = "/var/www/pages/feed"
|
||||
SYSROOT_OSTREE = "/sysroot/ostree/repo"
|
||||
OSTREE_BASE_DEPLOYMENT_DIR = "/ostree/deploy/debian/deploy/"
|
||||
PATCH_SCRIPTS_STAGING_DIR = "/run/patching/patch-scripts"
|
||||
|
||||
ENABLE_DEV_CERTIFICATE_PATCH_IDENTIFIER = 'ENABLE_DEV_CERTIFICATE'
|
||||
|
||||
|
|
|
@ -5,6 +5,8 @@ SPDX-License-Identifier: Apache-2.0
|
|||
|
||||
"""
|
||||
import logging
|
||||
import os.path
|
||||
import sh
|
||||
import subprocess
|
||||
|
||||
from cgcs_patch import constants
|
||||
|
@ -210,3 +212,51 @@ def create_deployment():
|
|||
% (e.returncode, e.stderr.decode("utf-8"))
|
||||
LOG.info(info_msg)
|
||||
raise OSTreeCommandFail(msg)
|
||||
|
||||
|
||||
def fetch_pending_deployment():
|
||||
"""
|
||||
Fetch the deployment ID of the pending deployment
|
||||
:return: The deployment ID of the pending deployment
|
||||
"""
|
||||
|
||||
cmd = "ostree admin status |grep pending |awk '{printf $2}'"
|
||||
|
||||
try:
|
||||
output = subprocess.run(cmd, shell=True, check=True, capture_output=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
msg = "Failed to fetch ostree admin status."
|
||||
info_msg = "OSTree Admin Status Error: return code: %s , Output: %s" \
|
||||
% (e.returncode, e.stderr.decode("utf-8"))
|
||||
LOG.info(info_msg)
|
||||
raise OSTreeCommandFail(msg)
|
||||
|
||||
# Store the output of the above command in a string
|
||||
pending_deployment = output.stdout.decode('utf-8')
|
||||
|
||||
return pending_deployment
|
||||
|
||||
|
||||
def mount_new_deployment(deployment_dir):
|
||||
"""
|
||||
Unmount /usr and /etc from the file system and remount it to directory
|
||||
<depoyment_dir>/usr and <depoyment_dir>/etc respectively
|
||||
:param deployment_dir: a path on the filesystem which points to the pending
|
||||
deployment
|
||||
example: /ostree/deploy/debian/deploy/<deployment_id>
|
||||
"""
|
||||
try:
|
||||
if os.path.ismount("/usr"):
|
||||
sh.umount("-l", "/usr")
|
||||
if os.path.ismount("/etc"):
|
||||
sh.umount("-l", "/etc")
|
||||
new_usr_mount_dir = "%s/usr" % (deployment_dir)
|
||||
new_etc_mount_dir = "%s/etc" % (deployment_dir)
|
||||
sh.mount("--bind", "-o", "ro,noatime", new_usr_mount_dir, "/usr")
|
||||
sh.mount("--bind", "-o", "ro,noatime", new_etc_mount_dir, "/etc")
|
||||
except sh.ErrorReturnCode as e:
|
||||
msg = "Failed to re-mount /usr and /etc."
|
||||
info_msg = "OSTree Deployment Mount Error: Output: %s" \
|
||||
% (e.stderr.decode("utf-8"))
|
||||
LOG.info(info_msg)
|
||||
raise OSTreeCommandFail(msg)
|
||||
|
|
|
@ -239,6 +239,9 @@ class PatchMessageAgentInstallReq(messages.PatchMessage):
|
|||
global pa
|
||||
resp = PatchMessageAgentInstallResp()
|
||||
|
||||
if not self.force:
|
||||
setflag(node_is_patched_rr_file)
|
||||
|
||||
if not os.path.exists(node_is_locked_file):
|
||||
if self.force:
|
||||
LOG.info("Installing on unlocked node, with force option")
|
||||
|
@ -406,11 +409,9 @@ class PatchAgent(PatchService):
|
|||
|
||||
try:
|
||||
# Create insvc patch directories
|
||||
if os.path.exists(insvc_patch_scripts):
|
||||
shutil.rmtree(insvc_patch_scripts, ignore_errors=True)
|
||||
if os.path.exists(insvc_patch_flags):
|
||||
shutil.rmtree(insvc_patch_flags, ignore_errors=True)
|
||||
if not os.path.exists(insvc_patch_scripts):
|
||||
os.makedirs(insvc_patch_scripts, 0o700)
|
||||
if not os.path.exists(insvc_patch_flags):
|
||||
os.makedirs(insvc_patch_flags, 0o700)
|
||||
except Exception:
|
||||
LOG.exception("Failed to create in-service patch directories")
|
||||
|
@ -459,15 +460,19 @@ class PatchAgent(PatchService):
|
|||
LOG.info("Disallowing patch-scripts. Treating as reboot-required")
|
||||
setflag(node_is_patched_rr_file)
|
||||
else:
|
||||
LOG.info("Running in-service patch-scripts")
|
||||
LOG.info("Mounting the new deployment")
|
||||
try:
|
||||
pending_deployment = ostree_utils.fetch_pending_deployment()
|
||||
deployment_dir = constants.OSTREE_BASE_DEPLOYMENT_DIR + pending_deployment
|
||||
ostree_utils.mount_new_deployment(deployment_dir)
|
||||
LOG.info("Running in-service patch-scripts")
|
||||
subprocess.check_output(run_insvc_patch_scripts_cmd, stderr=subprocess.STDOUT)
|
||||
|
||||
# Clear the node_is_patched flag, since we've handled it in-service
|
||||
clearflag(node_is_patched_file)
|
||||
self.node_is_patched = False
|
||||
except subprocess.CalledProcessError as e:
|
||||
LOG.exception("In-Service patch scripts failed")
|
||||
LOG.exception("In-Service patch installation failed")
|
||||
LOG.error("Command output: %s", e.output)
|
||||
success = False
|
||||
|
||||
|
|
|
@ -32,6 +32,7 @@ from cgcs_patch.patch_functions import committed_dir
|
|||
from cgcs_patch.patch_functions import PatchFile
|
||||
from cgcs_patch.patch_functions import package_dir
|
||||
from cgcs_patch.patch_functions import repo_dir
|
||||
from cgcs_patch.patch_functions import root_scripts_dir
|
||||
from cgcs_patch.patch_functions import semantics_dir
|
||||
from cgcs_patch.patch_functions import SW_VERSION
|
||||
from cgcs_patch.patch_functions import root_package_dir
|
||||
|
@ -833,6 +834,23 @@ class PatchController(PatchService):
|
|||
ostree_tar_filename = "%s/%s-software.tar" % (ostree_tar_dir, patch_id)
|
||||
return ostree_tar_filename
|
||||
|
||||
def delete_restart_script(self, patch_id):
|
||||
'''
|
||||
Deletes the restart script (if any) associated with the patch
|
||||
:param patch_id: The patch ID
|
||||
'''
|
||||
if not self.patch_data.metadata[patch_id]["restart_script"]:
|
||||
return
|
||||
|
||||
restart_script_path = "%s/%s" % (root_scripts_dir, self.patch_data.metadata[patch_id]["restart_script"])
|
||||
try:
|
||||
# Delete the metadata
|
||||
os.remove(restart_script_path)
|
||||
except OSError:
|
||||
msg = "Failed to remove restart script for %s" % patch_id
|
||||
LOG.exception(msg)
|
||||
raise PatchError(msg)
|
||||
|
||||
def get_repo_filename(self, patch_sw_version, contentname):
|
||||
contentfile = self.get_store_filename(patch_sw_version, contentname)
|
||||
if not os.path.isfile(contentfile):
|
||||
|
@ -1093,7 +1111,7 @@ class PatchController(PatchService):
|
|||
LOG.exception("Failure during commit consistency check for %s.", patch_id)
|
||||
|
||||
if self.patch_data.contents[patch_id]["base"]["commit"] != latest_commit:
|
||||
msg = "The base commit %s for %s does not match the latest commit %s" \
|
||||
msg = "The base commit %s for %s does not match the latest commit %s " \
|
||||
"on this system." \
|
||||
% (self.patch_data.contents[patch_id]["base"]["commit"],
|
||||
patch_id,
|
||||
|
@ -1387,6 +1405,7 @@ class PatchController(PatchService):
|
|||
LOG.exception(msg)
|
||||
raise MetadataFail(msg)
|
||||
|
||||
self.delete_restart_script(patch_id)
|
||||
self.patch_data.delete_patch(patch_id)
|
||||
msg = "%s has been deleted" % patch_id
|
||||
LOG.info(msg)
|
||||
|
@ -1941,6 +1960,42 @@ class PatchController(PatchService):
|
|||
|
||||
return rc
|
||||
|
||||
def copy_restart_scripts(self):
|
||||
with self.patch_data_lock:
|
||||
for patch_id in self.patch_data.metadata:
|
||||
if (self.patch_data.metadata[patch_id]["patchstate"] in
|
||||
[constants.PARTIAL_APPLY, constants.PARTIAL_REMOVE]) \
|
||||
and self.patch_data.metadata[patch_id]["restart_script"]:
|
||||
try:
|
||||
restart_script_name = self.patch_data.metadata[patch_id]["restart_script"]
|
||||
restart_script_path = "%s/%s" \
|
||||
% (root_scripts_dir, restart_script_name)
|
||||
dest_path = constants.PATCH_SCRIPTS_STAGING_DIR
|
||||
dest_script_file = "%s/%s" \
|
||||
% (constants.PATCH_SCRIPTS_STAGING_DIR, restart_script_name)
|
||||
if not os.path.exists(dest_path):
|
||||
os.makedirs(dest_path, 0o700)
|
||||
shutil.copyfile(restart_script_path, dest_script_file)
|
||||
os.chmod(dest_script_file, 0o700)
|
||||
msg = "Creating restart script for %s" % patch_id
|
||||
LOG.info(msg)
|
||||
except shutil.Error:
|
||||
msg = "Failed to copy the restart script for %s" % patch_id
|
||||
LOG.exception(msg)
|
||||
raise PatchError(msg)
|
||||
elif self.patch_data.metadata[patch_id]["restart_script"]:
|
||||
try:
|
||||
restart_script_name = self.patch_data.metadata[patch_id]["restart_script"]
|
||||
restart_script_path = "%s/%s" \
|
||||
% (constants.PATCH_SCRIPTS_STAGING_DIR, restart_script_name)
|
||||
if os.path.exists(restart_script_path):
|
||||
os.remove(restart_script_path)
|
||||
msg = "Removing restart script for %s" % patch_id
|
||||
LOG.info(msg)
|
||||
except shutil.Error:
|
||||
msg = "Failed to delete the restart script for %s" % patch_id
|
||||
LOG.exception(msg)
|
||||
|
||||
def patch_host_install(self, host_ip, force, async_req=False):
|
||||
msg_info = ""
|
||||
msg_warning = ""
|
||||
|
@ -1971,6 +2026,7 @@ class PatchController(PatchService):
|
|||
if self.allow_insvc_patching:
|
||||
LOG.info("Allowing in-service patching")
|
||||
force = True
|
||||
self.copy_restart_scripts()
|
||||
|
||||
self.hosts[ip].install_pending = True
|
||||
self.hosts[ip].install_status = False
|
||||
|
|
|
@ -48,6 +48,7 @@ repo_root_dir = "/var/www/pages/updates"
|
|||
repo_dir = {SW_VERSION: "%s/rel-%s" % (repo_root_dir, SW_VERSION)}
|
||||
|
||||
root_package_dir = "%s/packages" % patch_dir
|
||||
root_scripts_dir = "/etc/patching/patch-scripts"
|
||||
package_dir = {SW_VERSION: "%s/%s" % (root_package_dir, SW_VERSION)}
|
||||
|
||||
logfile = "/var/log/patching.log"
|
||||
|
@ -335,6 +336,7 @@ class PatchData(object):
|
|||
"summary",
|
||||
"description",
|
||||
"install_instructions",
|
||||
"restart_script",
|
||||
"warnings",
|
||||
"apply_active_release_only"]:
|
||||
value = root.findtext(key)
|
||||
|
@ -615,19 +617,21 @@ class PatchFile(object):
|
|||
# Open the patch file and extract the contents to the current dir
|
||||
tar = tarfile.open(path, "r:gz")
|
||||
|
||||
filelist = ["metadata.tar", "software.tar"]
|
||||
if "semantics.tar" in [f.name for f in tar.getmembers()]:
|
||||
filelist.append("semantics.tar")
|
||||
filelist = []
|
||||
for f in tar.getmembers():
|
||||
filelist.append(f.name)
|
||||
|
||||
if detached_signature_file not in filelist:
|
||||
msg = "Patch not signed"
|
||||
LOG.warning(msg)
|
||||
|
||||
for f in filelist:
|
||||
tar.extract(f)
|
||||
|
||||
tar.extract("signature")
|
||||
try:
|
||||
tar.extract(detached_signature_file)
|
||||
except KeyError:
|
||||
msg = "Patch has not been signed"
|
||||
LOG.warning(msg)
|
||||
# Filelist used for signature validation and verification
|
||||
sig_filelist = ["metadata.tar", "software.tar"]
|
||||
if "semantics.tar" in filelist:
|
||||
sig_filelist.append("semantics.tar")
|
||||
|
||||
# Verify the data integrity signature first
|
||||
sigfile = open("signature", "r")
|
||||
|
@ -635,7 +639,7 @@ class PatchFile(object):
|
|||
sigfile.close()
|
||||
|
||||
expected_sig = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
|
||||
for f in filelist:
|
||||
for f in sig_filelist:
|
||||
sig ^= get_md5(f)
|
||||
|
||||
if sig != expected_sig:
|
||||
|
@ -646,7 +650,7 @@ class PatchFile(object):
|
|||
# Verify detached signature
|
||||
if os.path.exists(detached_signature_file):
|
||||
sig_valid = verify_files(
|
||||
filelist,
|
||||
sig_filelist,
|
||||
detached_signature_file,
|
||||
cert_type=cert_type)
|
||||
if sig_valid is True:
|
||||
|
@ -850,6 +854,13 @@ class PatchFile(object):
|
|||
shutil.move("software.tar",
|
||||
"%s/%s-software.tar" % (abs_ostree_tar_dir, patch_id))
|
||||
|
||||
if thispatch.metadata[patch_id]["restart_script"]:
|
||||
if not os.path.exists(root_scripts_dir):
|
||||
os.makedirs(root_scripts_dir)
|
||||
restart_script_name = thispatch.metadata[patch_id]["restart_script"]
|
||||
shutil.move(restart_script_name,
|
||||
"%s/%s" % (root_scripts_dir, restart_script_name))
|
||||
|
||||
except PatchValidationFailure as e:
|
||||
raise e
|
||||
except PatchMismatchFailure as e:
|
||||
|
|
|
@ -293,7 +293,7 @@ zope=no
|
|||
# List of members which are set dynamically and missed by pylint inference
|
||||
# system, and so shouldn't trigger E0201 when accessed. Python regular
|
||||
# expressions are accepted.
|
||||
generated-members=REQUEST,acl_users,aq_parent
|
||||
generated-members=REQUEST,acl_users,aq_parent,sh.*
|
||||
|
||||
|
||||
[VARIABLES]
|
||||
|
|
|
@ -8,3 +8,4 @@ pecan
|
|||
pycryptodomex
|
||||
lxml
|
||||
requests_toolbelt
|
||||
sh
|
||||
|
|
|
@ -13,7 +13,8 @@ Build-Depends-Indep: python3-keystonemiddleware,
|
|||
python3-stestr,
|
||||
python3-testtools,
|
||||
python3-six,
|
||||
tsconfig
|
||||
tsconfig,
|
||||
python3-sh
|
||||
Standards-Version: 4.4.1
|
||||
|
||||
Package: cgcs-patch
|
||||
|
@ -45,6 +46,7 @@ Depends: ${python3:Depends},
|
|||
python3-lxml,
|
||||
python3-requests-toolbelt,
|
||||
python3-six,
|
||||
tsconfig
|
||||
tsconfig,
|
||||
python3-sh
|
||||
Description: Starlingx platfom patching (python3)
|
||||
Starlingx platform patching system python libraries
|
||||
|
|
Loading…
Reference in New Issue