Merge "Debian: In-service patch implementation"
This commit is contained in:
commit
def5e22b27
|
@ -46,6 +46,8 @@ OSTREE_REF = "starlingx"
|
||||||
OSTREE_REMOTE = "debian"
|
OSTREE_REMOTE = "debian"
|
||||||
FEED_OSTREE_BASE_DIR = "/var/www/pages/feed"
|
FEED_OSTREE_BASE_DIR = "/var/www/pages/feed"
|
||||||
SYSROOT_OSTREE = "/sysroot/ostree/repo"
|
SYSROOT_OSTREE = "/sysroot/ostree/repo"
|
||||||
|
OSTREE_BASE_DEPLOYMENT_DIR = "/ostree/deploy/debian/deploy/"
|
||||||
|
PATCH_SCRIPTS_STAGING_DIR = "/run/patching/patch-scripts"
|
||||||
|
|
||||||
ENABLE_DEV_CERTIFICATE_PATCH_IDENTIFIER = 'ENABLE_DEV_CERTIFICATE'
|
ENABLE_DEV_CERTIFICATE_PATCH_IDENTIFIER = 'ENABLE_DEV_CERTIFICATE'
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,8 @@ SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
|
import os.path
|
||||||
|
import sh
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
from cgcs_patch import constants
|
from cgcs_patch import constants
|
||||||
|
@ -210,3 +212,51 @@ def create_deployment():
|
||||||
% (e.returncode, e.stderr.decode("utf-8"))
|
% (e.returncode, e.stderr.decode("utf-8"))
|
||||||
LOG.info(info_msg)
|
LOG.info(info_msg)
|
||||||
raise OSTreeCommandFail(msg)
|
raise OSTreeCommandFail(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_pending_deployment():
|
||||||
|
"""
|
||||||
|
Fetch the deployment ID of the pending deployment
|
||||||
|
:return: The deployment ID of the pending deployment
|
||||||
|
"""
|
||||||
|
|
||||||
|
cmd = "ostree admin status |grep pending |awk '{printf $2}'"
|
||||||
|
|
||||||
|
try:
|
||||||
|
output = subprocess.run(cmd, shell=True, check=True, capture_output=True)
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
msg = "Failed to fetch ostree admin status."
|
||||||
|
info_msg = "OSTree Admin Status Error: return code: %s , Output: %s" \
|
||||||
|
% (e.returncode, e.stderr.decode("utf-8"))
|
||||||
|
LOG.info(info_msg)
|
||||||
|
raise OSTreeCommandFail(msg)
|
||||||
|
|
||||||
|
# Store the output of the above command in a string
|
||||||
|
pending_deployment = output.stdout.decode('utf-8')
|
||||||
|
|
||||||
|
return pending_deployment
|
||||||
|
|
||||||
|
|
||||||
|
def mount_new_deployment(deployment_dir):
|
||||||
|
"""
|
||||||
|
Unmount /usr and /etc from the file system and remount it to directory
|
||||||
|
<depoyment_dir>/usr and <depoyment_dir>/etc respectively
|
||||||
|
:param deployment_dir: a path on the filesystem which points to the pending
|
||||||
|
deployment
|
||||||
|
example: /ostree/deploy/debian/deploy/<deployment_id>
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if os.path.ismount("/usr"):
|
||||||
|
sh.umount("-l", "/usr")
|
||||||
|
if os.path.ismount("/etc"):
|
||||||
|
sh.umount("-l", "/etc")
|
||||||
|
new_usr_mount_dir = "%s/usr" % (deployment_dir)
|
||||||
|
new_etc_mount_dir = "%s/etc" % (deployment_dir)
|
||||||
|
sh.mount("--bind", "-o", "ro,noatime", new_usr_mount_dir, "/usr")
|
||||||
|
sh.mount("--bind", "-o", "ro,noatime", new_etc_mount_dir, "/etc")
|
||||||
|
except sh.ErrorReturnCode as e:
|
||||||
|
msg = "Failed to re-mount /usr and /etc."
|
||||||
|
info_msg = "OSTree Deployment Mount Error: Output: %s" \
|
||||||
|
% (e.stderr.decode("utf-8"))
|
||||||
|
LOG.info(info_msg)
|
||||||
|
raise OSTreeCommandFail(msg)
|
||||||
|
|
|
@ -239,6 +239,9 @@ class PatchMessageAgentInstallReq(messages.PatchMessage):
|
||||||
global pa
|
global pa
|
||||||
resp = PatchMessageAgentInstallResp()
|
resp = PatchMessageAgentInstallResp()
|
||||||
|
|
||||||
|
if not self.force:
|
||||||
|
setflag(node_is_patched_rr_file)
|
||||||
|
|
||||||
if not os.path.exists(node_is_locked_file):
|
if not os.path.exists(node_is_locked_file):
|
||||||
if self.force:
|
if self.force:
|
||||||
LOG.info("Installing on unlocked node, with force option")
|
LOG.info("Installing on unlocked node, with force option")
|
||||||
|
@ -406,12 +409,10 @@ class PatchAgent(PatchService):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Create insvc patch directories
|
# Create insvc patch directories
|
||||||
if os.path.exists(insvc_patch_scripts):
|
if not os.path.exists(insvc_patch_scripts):
|
||||||
shutil.rmtree(insvc_patch_scripts, ignore_errors=True)
|
os.makedirs(insvc_patch_scripts, 0o700)
|
||||||
if os.path.exists(insvc_patch_flags):
|
if not os.path.exists(insvc_patch_flags):
|
||||||
shutil.rmtree(insvc_patch_flags, ignore_errors=True)
|
os.makedirs(insvc_patch_flags, 0o700)
|
||||||
os.makedirs(insvc_patch_scripts, 0o700)
|
|
||||||
os.makedirs(insvc_patch_flags, 0o700)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
LOG.exception("Failed to create in-service patch directories")
|
LOG.exception("Failed to create in-service patch directories")
|
||||||
|
|
||||||
|
@ -459,15 +460,19 @@ class PatchAgent(PatchService):
|
||||||
LOG.info("Disallowing patch-scripts. Treating as reboot-required")
|
LOG.info("Disallowing patch-scripts. Treating as reboot-required")
|
||||||
setflag(node_is_patched_rr_file)
|
setflag(node_is_patched_rr_file)
|
||||||
else:
|
else:
|
||||||
LOG.info("Running in-service patch-scripts")
|
LOG.info("Mounting the new deployment")
|
||||||
try:
|
try:
|
||||||
|
pending_deployment = ostree_utils.fetch_pending_deployment()
|
||||||
|
deployment_dir = constants.OSTREE_BASE_DEPLOYMENT_DIR + pending_deployment
|
||||||
|
ostree_utils.mount_new_deployment(deployment_dir)
|
||||||
|
LOG.info("Running in-service patch-scripts")
|
||||||
subprocess.check_output(run_insvc_patch_scripts_cmd, stderr=subprocess.STDOUT)
|
subprocess.check_output(run_insvc_patch_scripts_cmd, stderr=subprocess.STDOUT)
|
||||||
|
|
||||||
# Clear the node_is_patched flag, since we've handled it in-service
|
# Clear the node_is_patched flag, since we've handled it in-service
|
||||||
clearflag(node_is_patched_file)
|
clearflag(node_is_patched_file)
|
||||||
self.node_is_patched = False
|
self.node_is_patched = False
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
LOG.exception("In-Service patch scripts failed")
|
LOG.exception("In-Service patch installation failed")
|
||||||
LOG.error("Command output: %s", e.output)
|
LOG.error("Command output: %s", e.output)
|
||||||
success = False
|
success = False
|
||||||
|
|
||||||
|
|
|
@ -32,6 +32,7 @@ from cgcs_patch.patch_functions import committed_dir
|
||||||
from cgcs_patch.patch_functions import PatchFile
|
from cgcs_patch.patch_functions import PatchFile
|
||||||
from cgcs_patch.patch_functions import package_dir
|
from cgcs_patch.patch_functions import package_dir
|
||||||
from cgcs_patch.patch_functions import repo_dir
|
from cgcs_patch.patch_functions import repo_dir
|
||||||
|
from cgcs_patch.patch_functions import root_scripts_dir
|
||||||
from cgcs_patch.patch_functions import semantics_dir
|
from cgcs_patch.patch_functions import semantics_dir
|
||||||
from cgcs_patch.patch_functions import SW_VERSION
|
from cgcs_patch.patch_functions import SW_VERSION
|
||||||
from cgcs_patch.patch_functions import root_package_dir
|
from cgcs_patch.patch_functions import root_package_dir
|
||||||
|
@ -833,6 +834,23 @@ class PatchController(PatchService):
|
||||||
ostree_tar_filename = "%s/%s-software.tar" % (ostree_tar_dir, patch_id)
|
ostree_tar_filename = "%s/%s-software.tar" % (ostree_tar_dir, patch_id)
|
||||||
return ostree_tar_filename
|
return ostree_tar_filename
|
||||||
|
|
||||||
|
def delete_restart_script(self, patch_id):
|
||||||
|
'''
|
||||||
|
Deletes the restart script (if any) associated with the patch
|
||||||
|
:param patch_id: The patch ID
|
||||||
|
'''
|
||||||
|
if not self.patch_data.metadata[patch_id]["restart_script"]:
|
||||||
|
return
|
||||||
|
|
||||||
|
restart_script_path = "%s/%s" % (root_scripts_dir, self.patch_data.metadata[patch_id]["restart_script"])
|
||||||
|
try:
|
||||||
|
# Delete the metadata
|
||||||
|
os.remove(restart_script_path)
|
||||||
|
except OSError:
|
||||||
|
msg = "Failed to remove restart script for %s" % patch_id
|
||||||
|
LOG.exception(msg)
|
||||||
|
raise PatchError(msg)
|
||||||
|
|
||||||
def get_repo_filename(self, patch_sw_version, contentname):
|
def get_repo_filename(self, patch_sw_version, contentname):
|
||||||
contentfile = self.get_store_filename(patch_sw_version, contentname)
|
contentfile = self.get_store_filename(patch_sw_version, contentname)
|
||||||
if not os.path.isfile(contentfile):
|
if not os.path.isfile(contentfile):
|
||||||
|
@ -1093,7 +1111,7 @@ class PatchController(PatchService):
|
||||||
LOG.exception("Failure during commit consistency check for %s.", patch_id)
|
LOG.exception("Failure during commit consistency check for %s.", patch_id)
|
||||||
|
|
||||||
if self.patch_data.contents[patch_id]["base"]["commit"] != latest_commit:
|
if self.patch_data.contents[patch_id]["base"]["commit"] != latest_commit:
|
||||||
msg = "The base commit %s for %s does not match the latest commit %s" \
|
msg = "The base commit %s for %s does not match the latest commit %s " \
|
||||||
"on this system." \
|
"on this system." \
|
||||||
% (self.patch_data.contents[patch_id]["base"]["commit"],
|
% (self.patch_data.contents[patch_id]["base"]["commit"],
|
||||||
patch_id,
|
patch_id,
|
||||||
|
@ -1387,6 +1405,7 @@ class PatchController(PatchService):
|
||||||
LOG.exception(msg)
|
LOG.exception(msg)
|
||||||
raise MetadataFail(msg)
|
raise MetadataFail(msg)
|
||||||
|
|
||||||
|
self.delete_restart_script(patch_id)
|
||||||
self.patch_data.delete_patch(patch_id)
|
self.patch_data.delete_patch(patch_id)
|
||||||
msg = "%s has been deleted" % patch_id
|
msg = "%s has been deleted" % patch_id
|
||||||
LOG.info(msg)
|
LOG.info(msg)
|
||||||
|
@ -1941,6 +1960,42 @@ class PatchController(PatchService):
|
||||||
|
|
||||||
return rc
|
return rc
|
||||||
|
|
||||||
|
def copy_restart_scripts(self):
|
||||||
|
with self.patch_data_lock:
|
||||||
|
for patch_id in self.patch_data.metadata:
|
||||||
|
if (self.patch_data.metadata[patch_id]["patchstate"] in
|
||||||
|
[constants.PARTIAL_APPLY, constants.PARTIAL_REMOVE]) \
|
||||||
|
and self.patch_data.metadata[patch_id]["restart_script"]:
|
||||||
|
try:
|
||||||
|
restart_script_name = self.patch_data.metadata[patch_id]["restart_script"]
|
||||||
|
restart_script_path = "%s/%s" \
|
||||||
|
% (root_scripts_dir, restart_script_name)
|
||||||
|
dest_path = constants.PATCH_SCRIPTS_STAGING_DIR
|
||||||
|
dest_script_file = "%s/%s" \
|
||||||
|
% (constants.PATCH_SCRIPTS_STAGING_DIR, restart_script_name)
|
||||||
|
if not os.path.exists(dest_path):
|
||||||
|
os.makedirs(dest_path, 0o700)
|
||||||
|
shutil.copyfile(restart_script_path, dest_script_file)
|
||||||
|
os.chmod(dest_script_file, 0o700)
|
||||||
|
msg = "Creating restart script for %s" % patch_id
|
||||||
|
LOG.info(msg)
|
||||||
|
except shutil.Error:
|
||||||
|
msg = "Failed to copy the restart script for %s" % patch_id
|
||||||
|
LOG.exception(msg)
|
||||||
|
raise PatchError(msg)
|
||||||
|
elif self.patch_data.metadata[patch_id]["restart_script"]:
|
||||||
|
try:
|
||||||
|
restart_script_name = self.patch_data.metadata[patch_id]["restart_script"]
|
||||||
|
restart_script_path = "%s/%s" \
|
||||||
|
% (constants.PATCH_SCRIPTS_STAGING_DIR, restart_script_name)
|
||||||
|
if os.path.exists(restart_script_path):
|
||||||
|
os.remove(restart_script_path)
|
||||||
|
msg = "Removing restart script for %s" % patch_id
|
||||||
|
LOG.info(msg)
|
||||||
|
except shutil.Error:
|
||||||
|
msg = "Failed to delete the restart script for %s" % patch_id
|
||||||
|
LOG.exception(msg)
|
||||||
|
|
||||||
def patch_host_install(self, host_ip, force, async_req=False):
|
def patch_host_install(self, host_ip, force, async_req=False):
|
||||||
msg_info = ""
|
msg_info = ""
|
||||||
msg_warning = ""
|
msg_warning = ""
|
||||||
|
@ -1971,6 +2026,7 @@ class PatchController(PatchService):
|
||||||
if self.allow_insvc_patching:
|
if self.allow_insvc_patching:
|
||||||
LOG.info("Allowing in-service patching")
|
LOG.info("Allowing in-service patching")
|
||||||
force = True
|
force = True
|
||||||
|
self.copy_restart_scripts()
|
||||||
|
|
||||||
self.hosts[ip].install_pending = True
|
self.hosts[ip].install_pending = True
|
||||||
self.hosts[ip].install_status = False
|
self.hosts[ip].install_status = False
|
||||||
|
|
|
@ -48,6 +48,7 @@ repo_root_dir = "/var/www/pages/updates"
|
||||||
repo_dir = {SW_VERSION: "%s/rel-%s" % (repo_root_dir, SW_VERSION)}
|
repo_dir = {SW_VERSION: "%s/rel-%s" % (repo_root_dir, SW_VERSION)}
|
||||||
|
|
||||||
root_package_dir = "%s/packages" % patch_dir
|
root_package_dir = "%s/packages" % patch_dir
|
||||||
|
root_scripts_dir = "/etc/patching/patch-scripts"
|
||||||
package_dir = {SW_VERSION: "%s/%s" % (root_package_dir, SW_VERSION)}
|
package_dir = {SW_VERSION: "%s/%s" % (root_package_dir, SW_VERSION)}
|
||||||
|
|
||||||
logfile = "/var/log/patching.log"
|
logfile = "/var/log/patching.log"
|
||||||
|
@ -335,6 +336,7 @@ class PatchData(object):
|
||||||
"summary",
|
"summary",
|
||||||
"description",
|
"description",
|
||||||
"install_instructions",
|
"install_instructions",
|
||||||
|
"restart_script",
|
||||||
"warnings",
|
"warnings",
|
||||||
"apply_active_release_only"]:
|
"apply_active_release_only"]:
|
||||||
value = root.findtext(key)
|
value = root.findtext(key)
|
||||||
|
@ -615,19 +617,21 @@ class PatchFile(object):
|
||||||
# Open the patch file and extract the contents to the current dir
|
# Open the patch file and extract the contents to the current dir
|
||||||
tar = tarfile.open(path, "r:gz")
|
tar = tarfile.open(path, "r:gz")
|
||||||
|
|
||||||
filelist = ["metadata.tar", "software.tar"]
|
filelist = []
|
||||||
if "semantics.tar" in [f.name for f in tar.getmembers()]:
|
for f in tar.getmembers():
|
||||||
filelist.append("semantics.tar")
|
filelist.append(f.name)
|
||||||
|
|
||||||
|
if detached_signature_file not in filelist:
|
||||||
|
msg = "Patch not signed"
|
||||||
|
LOG.warning(msg)
|
||||||
|
|
||||||
for f in filelist:
|
for f in filelist:
|
||||||
tar.extract(f)
|
tar.extract(f)
|
||||||
|
|
||||||
tar.extract("signature")
|
# Filelist used for signature validation and verification
|
||||||
try:
|
sig_filelist = ["metadata.tar", "software.tar"]
|
||||||
tar.extract(detached_signature_file)
|
if "semantics.tar" in filelist:
|
||||||
except KeyError:
|
sig_filelist.append("semantics.tar")
|
||||||
msg = "Patch has not been signed"
|
|
||||||
LOG.warning(msg)
|
|
||||||
|
|
||||||
# Verify the data integrity signature first
|
# Verify the data integrity signature first
|
||||||
sigfile = open("signature", "r")
|
sigfile = open("signature", "r")
|
||||||
|
@ -635,7 +639,7 @@ class PatchFile(object):
|
||||||
sigfile.close()
|
sigfile.close()
|
||||||
|
|
||||||
expected_sig = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
|
expected_sig = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
|
||||||
for f in filelist:
|
for f in sig_filelist:
|
||||||
sig ^= get_md5(f)
|
sig ^= get_md5(f)
|
||||||
|
|
||||||
if sig != expected_sig:
|
if sig != expected_sig:
|
||||||
|
@ -646,7 +650,7 @@ class PatchFile(object):
|
||||||
# Verify detached signature
|
# Verify detached signature
|
||||||
if os.path.exists(detached_signature_file):
|
if os.path.exists(detached_signature_file):
|
||||||
sig_valid = verify_files(
|
sig_valid = verify_files(
|
||||||
filelist,
|
sig_filelist,
|
||||||
detached_signature_file,
|
detached_signature_file,
|
||||||
cert_type=cert_type)
|
cert_type=cert_type)
|
||||||
if sig_valid is True:
|
if sig_valid is True:
|
||||||
|
@ -850,6 +854,13 @@ class PatchFile(object):
|
||||||
shutil.move("software.tar",
|
shutil.move("software.tar",
|
||||||
"%s/%s-software.tar" % (abs_ostree_tar_dir, patch_id))
|
"%s/%s-software.tar" % (abs_ostree_tar_dir, patch_id))
|
||||||
|
|
||||||
|
if thispatch.metadata[patch_id]["restart_script"]:
|
||||||
|
if not os.path.exists(root_scripts_dir):
|
||||||
|
os.makedirs(root_scripts_dir)
|
||||||
|
restart_script_name = thispatch.metadata[patch_id]["restart_script"]
|
||||||
|
shutil.move(restart_script_name,
|
||||||
|
"%s/%s" % (root_scripts_dir, restart_script_name))
|
||||||
|
|
||||||
except PatchValidationFailure as e:
|
except PatchValidationFailure as e:
|
||||||
raise e
|
raise e
|
||||||
except PatchMismatchFailure as e:
|
except PatchMismatchFailure as e:
|
||||||
|
|
|
@ -293,7 +293,7 @@ zope=no
|
||||||
# List of members which are set dynamically and missed by pylint inference
|
# List of members which are set dynamically and missed by pylint inference
|
||||||
# system, and so shouldn't trigger E0201 when accessed. Python regular
|
# system, and so shouldn't trigger E0201 when accessed. Python regular
|
||||||
# expressions are accepted.
|
# expressions are accepted.
|
||||||
generated-members=REQUEST,acl_users,aq_parent
|
generated-members=REQUEST,acl_users,aq_parent,sh.*
|
||||||
|
|
||||||
|
|
||||||
[VARIABLES]
|
[VARIABLES]
|
||||||
|
|
|
@ -8,3 +8,4 @@ pecan
|
||||||
pycryptodomex
|
pycryptodomex
|
||||||
lxml
|
lxml
|
||||||
requests_toolbelt
|
requests_toolbelt
|
||||||
|
sh
|
||||||
|
|
|
@ -13,7 +13,8 @@ Build-Depends-Indep: python3-keystonemiddleware,
|
||||||
python3-stestr,
|
python3-stestr,
|
||||||
python3-testtools,
|
python3-testtools,
|
||||||
python3-six,
|
python3-six,
|
||||||
tsconfig
|
tsconfig,
|
||||||
|
python3-sh
|
||||||
Standards-Version: 4.4.1
|
Standards-Version: 4.4.1
|
||||||
|
|
||||||
Package: cgcs-patch
|
Package: cgcs-patch
|
||||||
|
@ -45,6 +46,7 @@ Depends: ${python3:Depends},
|
||||||
python3-lxml,
|
python3-lxml,
|
||||||
python3-requests-toolbelt,
|
python3-requests-toolbelt,
|
||||||
python3-six,
|
python3-six,
|
||||||
tsconfig
|
tsconfig,
|
||||||
|
python3-sh
|
||||||
Description: Starlingx platfom patching (python3)
|
Description: Starlingx platfom patching (python3)
|
||||||
Starlingx platform patching system python libraries
|
Starlingx platform patching system python libraries
|
||||||
|
|
Loading…
Reference in New Issue