diff --git a/.zuul.yaml b/.zuul.yaml index b2a6836c..687b885b 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -51,6 +51,8 @@ name: stx-software-tox-py39 parent: tox-py39 nodeset: debian-bullseye + required-projects: + - starlingx/config files: - software/* vars: @@ -62,6 +64,8 @@ name: stx-software-tox-pylint parent: tox nodeset: debian-bullseye + required-projects: + - starlingx/config files: - software/* vars: diff --git a/software/debian/deb_folder/rules b/software/debian/deb_folder/rules index 21ea9eed..1588487e 100755 --- a/software/debian/deb_folder/rules +++ b/software/debian/deb_folder/rules @@ -2,6 +2,7 @@ export DH_VERBOSE = 1 export PYBUILD_NAME = software export PBR_VERSION=1.0.0 +PMONDIR := ${ROOT}/usr/share/starlingx/pmon.d ROOT := $(CURDIR)/debian/tmp @@ -12,5 +13,48 @@ override_dh_install: python3 setup.py install -f --install-layout=deb --root=$(ROOT) python3 setup.py bdist_wheel --universal -d $(CURDIR)/debian/$(PYBUILD_NAME)-wheels/usr/share/python-wheels install -d -m 755 $(ROOT)/usr/bin + install -d -m 755 $(ROOT)/usr/sbin + install -d -m 755 $(ROOT)/run + install -d -m 755 $(ROOT)/usr/share/bash-completion/completions + install -m 755 -d ${ROOT}/etc/goenabled.d + install -m 755 -d ${ROOT}/etc/init.d + install -m 755 -d ${ROOT}/etc/logrotate.d + install -m 755 -d ${ROOT}/etc/software + install -m 755 -d ${ROOT}/etc/software/software-scripts + install -m 755 -d ${ROOT}/lib/systemd/system + install -m 755 -d ${PMONDIR} + install -m 500 service-files/software-controller-daemon-init.sh \ + ${ROOT}/etc/init.d/software-controller-daemon + install -m 500 service-files/software-agent-init.sh \ + ${ROOT}/etc/init.d/software-agent + install -m 500 service-files/software-init.sh \ + ${ROOT}/etc/init.d/software + install -m 500 service-files/software-controller-init.sh \ + ${ROOT}/etc/init.d/software-controller + install -m 600 service-files/software.conf \ + ${ROOT}/etc/software/software.conf + install -m 644 service-files/policy.json \ + ${ROOT}/etc/software/policy.json + install -m 444 service-files/pmon-software-controller-daemon.conf \ + ${PMONDIR}/software-controller-daemon.conf + install -m 444 service-files/pmon-software-agent.conf \ + ${PMONDIR}/software-agent.conf + install -m 444 service-files/*.service \ + ${ROOT}/lib/systemd/system + install -m 444 service-files/software.completion \ + ${ROOT}/usr/share/bash-completion/completions/software + install -m 400 service-files/software-functions \ + ${ROOT}/etc/software/software-functions + install -m 444 service-files/software-tmpdirs.conf \ + ${ROOT}/run/software-tmpdirs.conf + install -m 500 service-files/run-software-scripts \ + ${ROOT}/usr/sbin/run-software-scripts + install -m 500 service-files/software-controller-daemon-restart \ + ${ROOT}/usr/sbin/software-controller-daemon-restart + install -m 500 service-files/software-agent-restart \ + ${ROOT}/usr/sbin/software-agent-restart + install -m 555 service-files/software_check_goenabled.sh \ + ${ROOT}/etc/goenabled.d/software_check_goenabled.sh + install -m 444 service-files/software.logrotate \ + ${ROOT}/etc/logrotate.d/software dh_install - diff --git a/software/debian/deb_folder/software.install b/software/debian/deb_folder/software.install index 78e37455..18506da5 100644 --- a/software/debian/deb_folder/software.install +++ b/software/debian/deb_folder/software.install @@ -1,2 +1,11 @@ -/usr/bin -/usr/lib/python*/dist-packages/* +etc/goenabled.d +etc/init.d +etc/logrotate.d +etc/software +lib/systemd/system +run/software-tmpdirs.conf +usr/bin +usr/lib/python*/dist-packages/* +usr/sbin +usr/share/ + diff --git a/software/pylint.rc b/software/pylint.rc index 4211134e..791e5191 100644 --- a/software/pylint.rc +++ b/software/pylint.rc @@ -31,7 +31,7 @@ extension-pkg-allow-list= # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. (This is an alternative name to extension-pkg-allow-list # for backward compatibility.) -extension-pkg-whitelist= +extension-pkg-whitelist=lxml # Return non-zero exit code if any of these messages/categories are detected, # even if score is above --fail-under value. Syntax same as enable. Messages @@ -420,7 +420,57 @@ confidence=HIGH, # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable= +# -Conventions- +# C0103 invalid-name +# C0114 missing-module-docstring +# C0115 missing-class-docstring +# C0116 missing-function-docstring +# C0201 consider-iterating-dictionary +# C0206 consider-using-dict-items +# C0209 consider-using-f-string +# C2801 unnecessary-dunder-call +# C0301 line-too-long +# C0302 too-many-lines +# C0325 superfluous-parens +# C0411 wrong-import-order +# C0415 import-outside-toplevel +# -Refactoring- +# R0205 useless-object-inheritance +# R0402 consider-using-from-import +# R0801 Similar lines in x files +# R0902 too-many-instance-attributes +# R0903 too-few-public-methods +# R0904 too-many-public-methods +# R0911 too-many-return-statements +# R0912 too-many-branches +# R0913 too-many-arguments +# R0914 too-many-locals +# R0915 too-many-statements +# R1702 too-many-nested-blocks +# R1705 no-else-return +# R1714 consider-using-in +# R1715 consider-using-get +# R1722 consider-using-sys-exit +# R1724 no-else-continue +# R1725 super-with-arguments +# R1732 consider-using-with +# R1735 use-dict-literal +# -Warnings- +# W0107 unnecessary-pass +# W0602 global-variable-not-assigned +# W0603 global-statement +# W0703 broad-except +# W0707 raise-missing-from +# W0719 broad-exception-raised +# W1505 deprecated-method +# W1514 unspecified-encoding +# W3101 missing-timeout +disable= C0103,C0114,C0115,C0116,C0201,C0206,C0209,C2801, + C0301,C0302,C0325,C0411,C0413,C0415, + R0205,R0402,R0801,R0902,R0903,R0904,R0911, + R0912,R0913,R0914,R0915,R1702,R1705,R1714, + R1715,R1722,R1724,R1725,R1732,R1735, + W0107,W0602,W0603,W0703,W0707,W0719,W1514,W3101 # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -547,7 +597,7 @@ contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members= +generated-members=sh # Tells whether to warn about missing members when the owner of the attribute # is inferred to be None. diff --git a/software/requirements.txt b/software/requirements.txt index de94602e..b45273f9 100644 --- a/software/requirements.txt +++ b/software/requirements.txt @@ -1,3 +1,12 @@ +keystoneauth1 +keystonemiddleware +lxml oslo.config -oslo.log +oslo.policy +oslo.serialization +netaddr pecan +pycryptodomex +requests_toolbelt +sh +WebOb diff --git a/software/service-files/pmon-software-agent.conf b/software/service-files/pmon-software-agent.conf new file mode 100644 index 00000000..8a80d59e --- /dev/null +++ b/software/service-files/pmon-software-agent.conf @@ -0,0 +1,19 @@ +[process] +process = software-agent +pidfile = /var/run/software-agent.pid +script = /etc/init.d/software-agent +style = lsb ; ocf or lsb +severity = major ; Process failure severity + ; critical : host is failed + ; major : host is degraded + ; minor : log is generated +restarts = 3 ; Number of back to back unsuccessful restarts before severity assertion +interval = 5 ; Number of seconds to wait between back-to-back unsuccessful restarts +debounce = 20 ; Number of seconds the process needs to run before declaring + ; it as running O.K. after a restart. + ; Time after which back-to-back restart count is cleared. +startuptime = 10 ; Seconds to wait after process start before starting the debounce monitor +mode = passive ; Monitoring mode: passive (default) or active + ; passive: process death monitoring (default: always) + ; active: heartbeat monitoring, i.e. request / response messaging + diff --git a/software/service-files/pmon-software-controller-daemon.conf b/software/service-files/pmon-software-controller-daemon.conf new file mode 100644 index 00000000..57eca49f --- /dev/null +++ b/software/service-files/pmon-software-controller-daemon.conf @@ -0,0 +1,19 @@ +[process] +process = software-controller-daemon +pidfile = /var/run/software-controller-daemon.pid +script = /etc/init.d/software-controller-daemon +style = lsb ; ocf or lsb +severity = major ; Process failure severity + ; critical : host is failed + ; major : host is degraded + ; minor : log is generated +restarts = 3 ; Number of back to back unsuccessful restarts before severity assertion +interval = 5 ; Number of seconds to wait between back-to-back unsuccessful restarts +debounce = 20 ; Number of seconds the process needs to run before declaring + ; it as running O.K. after a restart. + ; Time after which back-to-back restart count is cleared. +startuptime = 10 ; Seconds to wait after process start before starting the debounce monitor +mode = passive ; Monitoring mode: passive (default) or active + ; passive: process death monitoring (default: always) + ; active: heartbeat monitoring, i.e. request / response messaging + diff --git a/software/service-files/policy.json b/software/service-files/policy.json new file mode 100644 index 00000000..2c63c085 --- /dev/null +++ b/software/service-files/policy.json @@ -0,0 +1,2 @@ +{ +} diff --git a/software/service-files/run-software-scripts b/software/service-files/run-software-scripts new file mode 100644 index 00000000..eff3aa28 --- /dev/null +++ b/software/service-files/run-software-scripts @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +. /etc/software/software-functions + +declare SCRIPTS=$(find $PATCH_SCRIPTDIR -type f -executable | sort) +declare -i NUM_SCRIPTS=$(echo "$SCRIPTS" | wc -l) + +if [ $NUM_SCRIPTS -eq 0 ] +then + loginfo "No in-service patch scripts found." + exit 0 +fi + +loginfo "Running $NUM_SCRIPTS in-service patch scripts" + +declare SCRIPTLOG=/var/log/software-insvc.log +cat <>$SCRIPTLOG +############################################################ +`date "+%FT%T.%3N"`: Running $NUM_SCRIPTS in-service patch scripts: + +$SCRIPTS + +############################################################ +EOF + +declare -i FAILURES=0 +for cmd in $SCRIPTS +do + cat <>$SCRIPTLOG +############################################################ +`date "+%FT%T.%3N"`: Running $cmd + +EOF + + bash -x $cmd >>$SCRIPTLOG 2>&1 + rc=$? + if [ $rc -ne $PATCH_STATUS_OK ] + then + let -i FAILURES++ + fi + cat <>$SCRIPTLOG +`date "+%FT%T.%3N"`: Completed running $cmd (rc=$rc) +############################################################ + +EOF +done + +cat <>$SCRIPTLOG + +`date "+%FT%T.%3N"`: Completed running scripts with $FAILURES failures +############################################################ +EOF + +exit $FAILURES diff --git a/software/service-files/setup_software_repo b/software/service-files/setup_software_repo new file mode 100755 index 00000000..2e67bb10 --- /dev/null +++ b/software/service-files/setup_software_repo @@ -0,0 +1,141 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + +import getopt +import os +import platform +import shutil +import subprocess +import sys +import tempfile + +import software.software_functions as sf +import software.release_verify as pv +import sotware.constants as constants + +import logging +logging.getLogger('main_logger') +logging.basicConfig(level=logging.INFO) + +# Override the pv.dev_certificate_marker so we can verify signatures off-box +software_bindir = os.path.dirname(os.path.abspath(sys.argv[0])) +dev_cert_path = os.path.abspath(os.path.join(software_bindir, '../../enable-dev-patch/enable-dev-patch/dev_certificate_enable.bin')) + +pv.dev_certificate_marker = dev_cert_path + +def usage(): + print("Usage: %s -o ..." % os.path.basename(sys.argv[0])) + exit(1) + + +def main(): + try: + opts, remainder = getopt.getopt(sys.argv[1:], + 'o:', + ['output=']) + except getopt.GetoptError: + usage() + + output = None + + for opt, arg in opts: + if opt == "--output" or opt == '-o': + output = arg + + if output is None: + usage() + + sw_version = os.environ['PLATFORM_RELEASE'] + + allpatches = sf.PatchData() + + output = os.path.abspath(output) + + pkgdir = os.path.join(output, 'Packages') + datadir = os.path.join(output, 'metadata') + committed_dir = os.path.join(datadir, 'committed') + + if os.path.exists(output): + # Check to see if the expected structure already exists, + # maybe we're appending a patch. + if not os.path.exists(committed_dir) or not os.path.exists(pkgdir): + print("Packages or metadata dir missing from existing %s. Aborting..." % output) + exit(1) + + # Load the existing metadata + allpatches.load_all_metadata(committed_dir, constants.COMMITTED) + else: + os.mkdir(output, 0o755) + os.mkdir(datadir, 0o755) + os.mkdir(committed_dir, 0o755) + os.mkdir(pkgdir, 0o755) + + # Save the current directory, so we can chdir back after + orig_wd = os.getcwd() + + tmpdir = None + try: + for p in remainder: + fpath = os.path.abspath(p) + + # Create a temporary working directory + tmpdir = tempfile.mkdtemp(prefix="patchrepo_") + + # Change to the tmpdir + os.chdir(tmpdir) + + print("Parsing %s" % fpath) + sf.PatchFile.read_patch(fpath) + + thispatch = sf.PatchData() + patch_id = thispatch.parse_metadata("metadata.xml", constants.COMMITTED) + + if patch_id in allpatches.metadata: + print("Skipping %s as it's already in the repo" % patch_id) + # Change back to original working dir + os.chdir(orig_wd) + + shutil.rmtree(tmpdir) + tmpdir = None + + continue + + patch_sw_version = thispatch.query_line(patch_id, 'sw_version') + if patch_sw_version != sw_version: + raise Exception("%s is for release %s, not %s" % (patch_id, patch_sw_version, sw_version)) + + # Move the metadata to the "committed" dir, and the deb packages to the Packages dir + shutil.move('metadata.xml', os.path.join(committed_dir, "%s-metadata.xml" % patch_id)) + for f in thispatch.query_line(patch_id, 'contents'): + shutil.move(f, pkgdir) + + allpatches.add_patch(patch_id, thispatch) + + # Change back to original working dir + os.chdir(orig_wd) + + shutil.rmtree(tmpdir) + tmpdir = None + except: + if tmpdir is not None: + # Change back to original working dir + os.chdir(orig_wd) + + shutil.rmtree(tmpdir) + tmpdir = None + raise + + allpatches.gen_release_groups_xml(sw_version, output) + + # Purge unneeded deb pkgs + keep = {} + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/software/service-files/software-agent-init.sh b/software/service-files/software-agent-init.sh new file mode 100755 index 00000000..f21f4b94 --- /dev/null +++ b/software/service-files/software-agent-init.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# chkconfig: 345 26 30 + +### BEGIN INIT INFO +# Provides: software-agent +# Required-Start: $syslog +# Required-Stop: $syslog +# Default-Start: 2 3 5 +# Default-Stop: 0 1 6 +# Short-Description: software-agent +# Description: Provides the Unified Software Management Agent Daemon +### END INIT INFO + +DESC="software-agent" +DAEMON="/usr/bin/software-agent" +PIDFILE="/var/run/software-agent.pid" +PATCH_INSTALLING_FILE="/var/run/patch_installing" + +start() +{ + if [ -e $PIDFILE ]; then + PIDDIR=/proc/$(cat $PIDFILE) + if [ -d ${PIDDIR} ]; then + echo "$DESC already running." + exit 1 + else + echo "Removing stale PID file $PIDFILE" + rm -f $PIDFILE + fi + fi + + echo -n "Starting $DESC..." + + start-stop-daemon --start --quiet --background \ + --pidfile ${PIDFILE} --make-pidfile --exec ${DAEMON} + + if [ $? -eq 0 ]; then + echo "done." + else + echo "failed." + fi +} + +stop() +{ + if [ -f $PATCH_INSTALLING_FILE ]; then + echo "Patches are installing. Waiting for install to complete." + while [ -f $PATCH_INSTALLING_FILE ]; do + # Verify the agent is still running + pid=$(cat $PATCH_INSTALLING_FILE) + cat /proc/$pid/cmdline 2>/dev/null | grep -q $DAEMON + if [ $? -ne 0 ]; then + echo "Patch agent not running." + break + fi + sleep 1 + done + echo "Continuing with shutdown." + fi + + echo -n "Stopping $DESC..." + start-stop-daemon --stop --quiet --pidfile $PIDFILE + if [ $? -eq 0 ]; then + echo "done." + else + echo "failed." + fi + rm -f $PIDFILE +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart|force-reload) + stop + start + ;; + *) + echo "Usage: $0 {start|stop|force-reload|restart}" + exit 1 + ;; +esac + +exit 0 diff --git a/software/service-files/software-agent-restart b/software/service-files/software-agent-restart new file mode 100644 index 00000000..b0cc2858 --- /dev/null +++ b/software/service-files/software-agent-restart @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +. /etc/software/software-functions + +# +# Triggering a restart of the software daemons is done by +# creating a flag file and letting the daemon handle the restart. +# +loginfo "Requesting restart of software-agent" + +restart_software_agent_flag="/run/software/.restart.software-agent" +touch $restart_software_agent_flag + +exit 0 + diff --git a/software/service-files/software-agent.service b/software/service-files/software-agent.service new file mode 100644 index 00000000..94013aec --- /dev/null +++ b/software/service-files/software-agent.service @@ -0,0 +1,16 @@ +[Unit] +Description=Unified Software Management Agent +After=syslog.target network-online.target software.service +Before=pmon.service + +[Service] +Type=forking +User=root +ExecStart=/etc/init.d/software-agent start +ExecStop=/etc/init.d/software-agent stop +ExecReload=/etc/init.d/software-agent restart +PIDFile=/var/run/software-agent.pid + +[Install] +WantedBy=multi-user.target + diff --git a/software/service-files/software-controller-daemon-init.sh b/software/service-files/software-controller-daemon-init.sh new file mode 100755 index 00000000..30257a3b --- /dev/null +++ b/software/service-files/software-controller-daemon-init.sh @@ -0,0 +1,78 @@ +#!/bin/sh +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# chkconfig: 345 25 30 + +### BEGIN INIT INFO +# Provides: software-controller-daemon +# Required-Start: $syslog +# Required-Stop: $syslog +# Default-Start: 2 3 5 +# Default-Stop: 0 1 6 +# Short-Description: software-controller-daemon +# Description: Provides the Unified Software Patch Controller Daemon +### END INIT INFO + +DESC="software-controller-daemon" +DAEMON="/usr/bin/software-controller-daemon" +PIDFILE="/var/run/software-controller-daemon.pid" + +start() +{ + if [ -e $PIDFILE ]; then + PIDDIR=/proc/$(cat $PIDFILE) + if [ -d ${PIDDIR} ]; then + echo "$DESC already running." + exit 1 + else + echo "Removing stale PID file $PIDFILE" + rm -f $PIDFILE + fi + fi + + echo -n "Starting $DESC..." + + start-stop-daemon --start --quiet --background \ + --pidfile ${PIDFILE} --make-pidfile --exec ${DAEMON} + + if [ $? -eq 0 ]; then + echo "done." + else + echo "failed." + fi +} + +stop() +{ + echo -n "Stopping $DESC..." + start-stop-daemon --stop --quiet --pidfile $PIDFILE + if [ $? -eq 0 ]; then + echo "done." + else + echo "failed." + fi + rm -f $PIDFILE +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart|force-reload) + stop + start + ;; + *) + echo "Usage: $0 {start|stop|force-reload|restart}" + exit 1 + ;; +esac + +exit 0 diff --git a/software/service-files/software-controller-daemon-restart b/software/service-files/software-controller-daemon-restart new file mode 100644 index 00000000..fae4d0c7 --- /dev/null +++ b/software/service-files/software-controller-daemon-restart @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +. /etc/software/software-functions + +# +# Triggering a restart of the software daemons is done by +# creating a flag file and letting the daemon handle the restart. +# +loginfo "Requesting restart of software-controller" + +restart_software_controller_flag="/run/software/.restart.software-controller" +touch $restart_software_controller_flag + +exit 0 + diff --git a/software/service-files/software-controller-daemon.service b/software/service-files/software-controller-daemon.service new file mode 100644 index 00000000..9f373f27 --- /dev/null +++ b/software/service-files/software-controller-daemon.service @@ -0,0 +1,16 @@ +[Unit] +Description=Unified Software Management Controller Daemon +After=syslog.target network-online.target software.service software-controller.service +Before=pmon.service + +[Service] +Type=forking +User=root +ExecStart=/etc/init.d/software-controller-daemon start +ExecStop=/etc/init.d/software-controller-daemon stop +ExecReload=/etc/init.d/software-controller-daemon restart +PIDFile=/var/run/software-controller-daemon.pid + +[Install] +WantedBy=multi-user.target + diff --git a/software/service-files/software-controller-init.sh b/software/service-files/software-controller-init.sh new file mode 100644 index 00000000..66a76dce --- /dev/null +++ b/software/service-files/software-controller-init.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# StarlingX Patching Controller setup +# chkconfig: 345 20 24 +# description: CGCS Patching Controller init script + +### BEGIN INIT INFO +# Provides: software-controller +# Required-Start: $syslog +# Required-Stop: $syslog +# Default-Start: 2 3 5 +# Default-Stop: 0 1 6 +# Short-Description: software-controller +# Description: Provides the Unified Software Management Controller Daemon +### END INIT INFO + +. /usr/bin/tsconfig + +NAME=$(basename $0) + +REPO_ID=updates +REPO_ROOT=/var/www/pages/${REPO_ID} +REPO_DIR=${REPO_ROOT}/rel-${SW_VERSION} +GROUPS_FILE=$REPO_DIR/comps.xml +PATCHING_DIR=/opt/software + +logfile=/var/log/software.log + +function LOG { + logger "$NAME: $*" + echo "`date "+%FT%T.%3N"`: $NAME: $*" >> $logfile +} + +function LOG_TO_FILE { + echo "`date "+%FT%T.%3N"`: $NAME: $*" >> $logfile +} + +function do_setup { + # Does the repo exist? + if [ ! -d $REPO_DIR ]; then + LOG "Creating repo." + mkdir -p $REPO_DIR + + # The original Centos code would create the groups and call createrepo + # todo(jcasteli): determine if the ostree code needs a setup also + fi + + if [ ! -d $PATCHING_DIR ]; then + LOG "Creating $PATCHING_DIR" + mkdir -p $PATCHING_DIR + fi + + # If we can ping the active controller, sync the repos + LOG_TO_FILE "ping -c 1 -w 1 controller" + ping -c 1 -w 1 controller >> $logfile 2>&1 || ping6 -c 1 -w 1 controller >> $logfile 2>&1 + if [ $? -ne 0 ]; then + LOG "Cannot ping controller. Nothing to do" + return 0 + fi + + # Sync the software dir + LOG_TO_FILE "rsync -acv --delete rsync://controller/software/ ${PATCHING_DIR}/" + rsync -acv --delete rsync://controller/software/ ${PATCHING_DIR}/ >> $logfile 2>&1 + + # Sync the repo dir + LOG_TO_FILE "rsync -acv --delete rsync://controller/repo/ ${REPO_ROOT}/" + rsync -acv --delete rsync://controller/repo/ ${REPO_ROOT}/ >> $logfile 2>&1 +} + +case "$1" in + start) + do_setup + ;; + status) + ;; + stop) + # Nothing to do here + ;; + restart) + do_setup + ;; + *) + echo "Usage: $0 {status|start|stop|restart}" + exit 1 +esac + +exit 0 + diff --git a/software/service-files/software-controller.service b/software/service-files/software-controller.service new file mode 100644 index 00000000..91417338 --- /dev/null +++ b/software/service-files/software-controller.service @@ -0,0 +1,14 @@ +[Unit] +Description=Unified Software Management Controller +After=syslog.service network-online.target software.service +Before=software-agent.service software-controller-daemon.service + +[Service] +Type=oneshot +User=root +ExecStart=/etc/init.d/software-controller start +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target + diff --git a/software/service-files/software-functions b/software/service-files/software-functions new file mode 100644 index 00000000..888fadbe --- /dev/null +++ b/software/service-files/software-functions @@ -0,0 +1,52 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# This bash source file provides variables and functions that +# may be used by in-service patch scripts. +# + +# Source platform.conf, for nodetype and subfunctions +. /etc/platform/platform.conf + +declare PATCH_SCRIPTDIR=/run/software/software-scripts +declare PATCH_FLAGDIR=/run/software/software-flags +declare -i PATCH_STATUS_OK=0 +declare -i PATCH_STATUS_FAILED=1 + +declare logfile=/var/log/software.log +declare NAME=$(basename $0) + +function loginfo() +{ + echo "`date "+%FT%T.%3N"`: $NAME: $*" >> $logfile +} + +function is_controller() +{ + [[ $nodetype == "controller" ]] +} + +function is_worker() +{ + [[ $nodetype == "worker" ]] +} + +function is_storage() +{ + [[ $nodetype == "storage" ]] +} + +function is_cpe() +{ + [[ $nodetype == "controller" && $subfunction =~ worker ]] +} + +function is_locked() +{ + test -f /var/run/.node_locked +} + diff --git a/software/service-files/software-init.sh b/software/service-files/software-init.sh new file mode 100644 index 00000000..9fabd484 --- /dev/null +++ b/software/service-files/software-init.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Unified Software Management +# chkconfig: 345 20 23 +# description: StarlingX Unified Software Management init script + +### BEGIN INIT INFO +# Provides: software +# Required-Start: $syslog +# Required-Stop: $syslog +# Default-Start: 2 3 5 +# Default-Stop: 0 1 6 +# Short-Description: software +# Description: Provides the Unified Software Management component +### END INIT INFO + +NAME=$(basename $0) + +. /usr/bin/tsconfig +. /etc/platform/platform.conf + +logfile=/var/log/software.log +patch_failed_file=/var/run/patch_install_failed +patched_during_init=/etc/software/.patched_during_init + +# if the system has never been bootstrapped, system_mode is not set +# treat a non bootstrapped system like it is simplex +# and manually manage lighttpd, etc.. +if [ "${system_mode}" = "" ]; then + system_mode="simplex" +fi + +function LOG_TO_FILE { + echo "`date "+%FT%T.%3N"`: $NAME: $*" >> $logfile +} + +function check_for_rr_patch { + if [ -f /var/run/node_is_patched_rr ]; then + if [ ! -f ${patched_during_init} ]; then + echo + echo "Node has been patched and requires an immediate reboot." + echo + LOG_TO_FILE "Node has been patched, with reboot-required flag set. Rebooting" + touch ${patched_during_init} + /sbin/reboot + else + echo + echo "Node has been patched during init a second consecutive time. Skipping reboot due to possible error" + echo + LOG_TO_FILE "Node has been patched during init a second consecutive time. Skipping reboot due to possible error" + touch ${patch_failed_file} + rm -f ${patched_during_init} + exit 1 + fi + else + rm -f ${patched_during_init} + fi +} + +function check_install_uuid { + # Check whether our installed load matches the active controller + CONTROLLER_UUID=`curl -sf http://controller:${http_port}/feed/rel-${SW_VERSION}/install_uuid` + if [ $? -ne 0 ]; then + if [ "$HOSTNAME" = "controller-1" ]; then + # If we're on controller-1, controller-0 may not have the install_uuid + # matching this release, if we're in an upgrade. If the file doesn't exist, + # bypass this check + return 0 + fi + + LOG_TO_FILE "Unable to retrieve installation uuid from active controller" + echo "Unable to retrieve installation uuid from active controller" + return 1 + fi + + if [ "$INSTALL_UUID" != "$CONTROLLER_UUID" ]; then + LOG_TO_FILE "This node is running a different load than the active controller and must be reinstalled" + echo "This node is running a different load than the active controller and must be reinstalled" + return 1 + fi + + return 0 +} + +# Check for installation failure +if [ -f /etc/platform/installation_failed ] ; then + LOG_TO_FILE "/etc/platform/installation_failed flag is set. Aborting." + echo "$(basename $0): Detected installation failure. Aborting." + exit 1 +fi + +# For AIO-SX, abort if config is not yet applied and this is running in init +if [ "${system_mode}" = "simplex" -a ! -f ${INITIAL_CONTROLLER_CONFIG_COMPLETE} -a "$1" = "start" ]; then + LOG_TO_FILE "Config is not yet applied. Skipping init patching" + exit 0 +fi + +# If the management interface is bonded, it may take some time +# before communications can be properly setup. +# Allow up to $DELAY_SEC seconds to reach controller. +DELAY_SEC=120 +START=`date +%s` +FOUND=0 +while [ $(date +%s) -lt $(( ${START} + ${DELAY_SEC} )) ]; do + LOG_TO_FILE "Waiting for controller to be pingable" + ping -c 1 controller > /dev/null 2>&1 || ping6 -c 1 controller > /dev/null 2>&1 + if [ $? -eq 0 ]; then + LOG_TO_FILE "controller is pingable" + FOUND=1 + break + fi + sleep 1 +done + +if [ ${FOUND} -eq 0 ]; then + # 'controller' is not available, just exit + LOG_TO_FILE "Unable to contact active controller (controller). Boot will continue." + exit 1 +fi + +RC=0 +case "$1" in + start) + if [ "${system_mode}" = "simplex" ]; then + # On a simplex CPE, we need to launch the http server first, + # before we can do the patch installation + LOG_TO_FILE "***** Launching lighttpd *****" + /etc/init.d/lighttpd start + + LOG_TO_FILE "***** Starting patch operation *****" + /usr/sbin/software-agent --install 2>>$logfile + if [ -f ${patch_failed_file} ]; then + RC=1 + LOG_TO_FILE "***** Patch operation failed *****" + fi + LOG_TO_FILE "***** Finished patch operation *****" + + LOG_TO_FILE "***** Shutting down lighttpd *****" + /etc/init.d/lighttpd stop + else + check_install_uuid + if [ $? -ne 0 ]; then + # The INSTALL_UUID doesn't match the active controller, so exit + exit 1 + fi + + LOG_TO_FILE "***** Starting patch operation *****" + /usr/sbin/software-agent --install 2>>$logfile + if [ -f ${patch_failed_file} ]; then + RC=1 + LOG_TO_FILE "***** Patch operation failed *****" + fi + LOG_TO_FILE "***** Finished patch operation *****" + fi + + check_for_rr_patch + ;; + stop) + # Nothing to do here + ;; + restart) + LOG_TO_FILE "***** Starting patch operation *****" + /usr/sbin/software-agent --install 2>>$logfile + if [ -f ${patch_failed_file} ]; then + RC=1 + LOG_TO_FILE "***** Patch operation failed *****" + fi + LOG_TO_FILE "***** Finished patch operation *****" + ;; + *) + echo "Usage: $0 {start|stop|restart}" + exit 1 +esac + +exit $RC + diff --git a/software/service-files/software-tmpdirs.conf b/software/service-files/software-tmpdirs.conf new file mode 100644 index 00000000..c550fe89 --- /dev/null +++ b/software/service-files/software-tmpdirs.conf @@ -0,0 +1,2 @@ +d /run/software 0700 root root - + diff --git a/software/service-files/software.completion b/software/service-files/software.completion new file mode 100644 index 00000000..a935c5f0 --- /dev/null +++ b/software/service-files/software.completion @@ -0,0 +1,153 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# This file provides bash-completion functionality for +# the unified software management CLI +# + +function _sw() +{ + COMPREPLY=() + local cur="${COMP_WORDS[COMP_CWORD]}" + local prev="${COMP_WORDS[COMP_CWORD-1]}" + local subcommand=${COMP_WORDS[1]} + + # + # The available software subcommands + # + local subcommands=" + release upload + release upload-dir + release delete + release list + release show + deploy create + deploy list + deploy precheck + deploy start + deploy host + deploy query + deploy activate + deploy complete + deploy abort + deploy host-rollback + is-applied + is-available + report-app-dependencies + query-app-dependencies + what-requires + " + if [ -f /etc/platform/.initial_config_complete ]; then + # Post-config, so the host-install commands are accessible + subcommands="${subcommands} deploy host" + else + # Pre-config, so the install-local command is accessible + subcommands="${subcommands} install-local" + fi + + # Appends the '/' when completing dir names + set mark-directories on + + if [ $COMP_CWORD -gt 1 ]; then + # + # Complete the arguments to the subcommands. + # + case "$subcommand" in + apply|delete|show|what-requires|is-applied|is-available) + # Query the list of known patches + local patches=$(software completion patches 2>/dev/null) + COMPREPLY=( $(compgen -W "${patches}" -- ${cur}) ) + return 0 + ;; + remove) + # Query the list of known patches + local patches=$(software completion patches 2>/dev/null) + COMPREPLY=( $(compgen -W "--skipappcheck ${patches}" -- ${cur}) ) + return 0 + ;; + host-install|host-install-async|drop-host) + if [ "${prev}" = "${subcommand}" -o "${prev}" = "--force" ]; then + # Query the list of known hosts + local names=$(software completion hosts 2>/dev/null) + COMPREPLY=( $(compgen -W "${names}" -- ${cur}) ) + else + # Only one host can be specified, so no more completion + COMPREPLY=( $(compgen -- ${cur}) ) + fi + return 0 + ;; + upload) + # Allow dirs and files with .patch extension for completion + COMPREPLY=( $(compgen -f -o plusdirs -X '!*.patch' -- ${cur}) ) + return 0 + ;; + upload-dir) + # Allow dirs only for completion + COMPREPLY=( $(compgen -d -- ${cur}) ) + return 0 + ;; + query) + if [ "${prev}" = "--release" ]; then + # If --release has been specified, provide installed releases for completion + local releases=$(/bin/ls -d /var/www/pages/feed/rel-* 2>/dev/null | sed 's#/var/www/pages/feed/rel-##') + COMPREPLY=( $(compgen -W "${releases}" -- ${cur}) ) + else + # --release is only completion option for query + COMPREPLY=( $(compgen -W "--release" -- ${cur}) ) + fi + return 0 + ;; + query-hosts|install-local) + # These subcommands have no options/arguments + COMPREPLY=( $(compgen -- ${cur}) ) + return 0 + ;; + query-dependencies) + # Query the list of known patches + local patches=$(software completion patches 2>/dev/null) + COMPREPLY=( $(compgen -W "--recursive ${patches}" -- ${cur}) ) + return 0 + ;; + commit) + if [ "${prev}" = "--release" ]; then + # If --release has been specified, provide installed releases for completion + local releases=$(/bin/ls -d /var/www/pages/feed/rel-* 2>/dev/null | sed 's#/var/www/pages/feed/rel-##') + COMPREPLY=( $(compgen -W "${releases}" -- ${cur}) ) + else + # Query the list of known patches + local patches=$(software completion patches 2>/dev/null) + COMPREPLY=( $(compgen -W "--all --dry-run --release ${patches}" -- ${cur}) ) + fi + return 0 + ;; + report-app-dependencies) + if [ "${prev}" = "${subcommand}" ]; then + COMPREPLY=( $(compgen -W "--app" -- ${cur}) ) + elif [ "${prev}" = "--app" ]; then + COMPREPLY= + else + local patches=$(software completion patches 2>/dev/null) + COMPREPLY=( $(compgen -W "${patches}" -- ${cur}) ) + fi + return 0 + ;; + query-app-dependencies) + return 0 + ;; + *) + ;; + esac + fi + + # Provide subcommands for completion + COMPREPLY=($(compgen -W "${subcommands}" -- ${cur})) + return 0 +} + +# Bind the above function to the software CLI +complete -F _sw -o filenames software + diff --git a/software/service-files/software.conf b/software/service-files/software.conf new file mode 100644 index 00000000..4c78897e --- /dev/null +++ b/software/service-files/software.conf @@ -0,0 +1,7 @@ +[runtime] +controller_multicast = 239.1.1.3 +agent_multicast = 239.1.1.4 +api_port = 5493 +controller_port = 5494 +agent_port = 5495 + diff --git a/software/service-files/software.logrotate b/software/service-files/software.logrotate new file mode 100644 index 00000000..702d2a01 --- /dev/null +++ b/software/service-files/software.logrotate @@ -0,0 +1,15 @@ +/var/log/software.log +/var/log/software-api.log +/var/log/software-insvc.log +{ + nodateext + size 10M + start 1 + rotate 10 + missingok + notifempty + compress + delaycompress + copytruncate +} + diff --git a/software/service-files/software.service b/software/service-files/software.service new file mode 100644 index 00000000..15893e90 --- /dev/null +++ b/software/service-files/software.service @@ -0,0 +1,15 @@ +[Unit] +Description=Unified Software Management +After=syslog.target network-online.target +Before=software-agent.service + +[Service] +Type=oneshot +User=root +ExecStart=/etc/init.d/software start +RemainAfterExit=yes +StandardOutput=journal+console +StandardError=journal+console + +[Install] +WantedBy=multi-user.target diff --git a/software/service-files/software_check_goenabled.sh b/software/service-files/software_check_goenabled.sh new file mode 100644 index 00000000..8ee967c7 --- /dev/null +++ b/software/service-files/software_check_goenabled.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Software "goenabled" check. +# If a minor software release version has been applied on this node, +# it is now out-of-date and should be rebooted. + +NAME=$(basename $0) +SYSTEM_CHANGED_FLAG=/var/run/node_is_patched + +logfile=/var/log/software.log + +function LOG { + logger "$NAME: $*" + echo "`date "+%FT%T.%3N"`: $NAME: $*" >> $logfile +} + +if [ -f $SYSTEM_CHANGED_FLAG ]; then + LOG "Node has been patched. Failing goenabled check." + exit 1 +fi + +exit 0 + diff --git a/software/setup.cfg b/software/setup.cfg index 02a8fdfd..9f18743f 100644 --- a/software/setup.cfg +++ b/software/setup.cfg @@ -24,8 +24,9 @@ packages = [entry_points] console_scripts = - software = software.cmd.shell:main - software-api = software.cmd.api:main + software = software.software_client:main + software-controller-daemon = software.software_controller:main + software-agent = software.software_agent:main [wheel] universal = 1 diff --git a/software/software/api/app.py b/software/software/api/app.py index 2ea68485..01a1680e 100644 --- a/software/software/api/app.py +++ b/software/software/api/app.py @@ -17,7 +17,7 @@ def get_pecan_config(): cfg_dict = { # todo(abailey): add server defaults to config "server": { - "port": "5490", + "port": "5496", "host": "127.0.0.1" }, "app": { @@ -52,3 +52,12 @@ def setup_app(pecan_config=None): guess_content_type_from_ext=pecan_config.app.guess_content_type_from_ext ) return app + + +class VersionSelectorApplication(object): + def __init__(self): + pc = get_pecan_config() + self.v1 = setup_app(pecan_config=pc) + + def __call__(self, environ, start_response): + return self.v1(environ, start_response) diff --git a/software/software/api/controllers/root.py b/software/software/api/controllers/root.py index b999f0e2..63beeefc 100644 --- a/software/software/api/controllers/root.py +++ b/software/software/api/controllers/root.py @@ -6,6 +6,36 @@ SPDX-License-Identifier: Apache-2.0 """ from pecan import expose +from software.exceptions import PatchError +from software.software_controller import pc + + +class PatchAPIController(object): + + @expose('json') + @expose('query.xml', content_type='application/xml') + def host_install_async(self, *args): + if len(list(args)) == 0: + return dict(error="Host must be specified for install") + force = False + if len(list(args)) > 1 and 'force' in list(args)[1:]: + force = True + + try: + result = pc.patch_host_install(list(args)[0], force, async_req=True) + except PatchError as e: + return dict(error="Error: %s" % str(e)) + + return result + + @expose('json') + def is_applied(self, *args): + return pc.is_applied(list(args)) + + @expose('json') + def is_available(self, *args): + return pc.is_available(list(args)) + class RootController: """pecan REST API root""" @@ -14,4 +44,7 @@ class RootController: @expose('json') def index(self): """index for the root""" - return {} + return "Unified Software Management API, Available versions: /v1" + + patch = PatchAPIController() + v1 = PatchAPIController() diff --git a/software/software/authapi/__init__.py b/software/software/authapi/__init__.py new file mode 100755 index 00000000..b80ec7b2 --- /dev/null +++ b/software/software/authapi/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from oslo_config import cfg + +API_SERVICE_OPTS = [ + cfg.StrOpt('auth_api_bind_ip', + default=None, + help='IP for the authenticated Unified Software Management API server to bind to'), + cfg.IntOpt('auth_api_port', + default=5497, + help='The port for the authenticated Unified Software Management API server'), + cfg.IntOpt('api_limit_max', + default=1000, + help='the maximum number of items returned in a single ' + 'response from a collection resource') +] + +CONF = cfg.CONF +opt_group = cfg.OptGroup(name='api', + title='Options for the patch-api service') +CONF.register_group(opt_group) +CONF.register_opts(API_SERVICE_OPTS) diff --git a/software/software/authapi/acl.py b/software/software/authapi/acl.py new file mode 100755 index 00000000..c48773c3 --- /dev/null +++ b/software/software/authapi/acl.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""Access Control Lists (ACL's) control access the API server.""" +from software.authapi import auth_token + +OPT_GROUP_NAME = 'keystone_authtoken' +OPT_GROUP_PROVIDER = 'keystonemiddleware.auth_token' + + +def install(app, conf, public_routes): + """Install ACL check on application. + + :param app: A WSGI application. + :param conf: Settings. Must include OPT_GROUP_NAME section. + :param public_routes: The list of the routes which will be allowed + access without authentication. + :return: The same WSGI application with ACL installed. + + """ + keystone_config = dict(conf.get(OPT_GROUP_NAME)) + return auth_token.AuthTokenMiddleware(app, + conf=keystone_config, + public_api_routes=public_routes) diff --git a/software/software/authapi/app.py b/software/software/authapi/app.py new file mode 100755 index 00000000..f737edef --- /dev/null +++ b/software/software/authapi/app.py @@ -0,0 +1,74 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + +from oslo_config import cfg +import pecan + +from software.authapi import acl +from software.authapi import config +from software.authapi import hooks +from software.authapi import policy + +auth_opts = [ + cfg.StrOpt('auth_strategy', + default='keystone', + help='Method to use for auth: noauth or keystone.'), +] + +CONF = cfg.CONF +CONF.register_opts(auth_opts) + + +def get_pecan_config(): + # Set up the pecan configuration + filename = config.__file__.replace('.pyc', '.py') + return pecan.configuration.conf_from_file(filename) + + +def setup_app(pecan_config=None, extra_hooks=None): + policy.init() + + app_hooks = [hooks.ConfigHook(), + hooks.ContextHook(pecan_config.app.acl_public_routes), + ] + if extra_hooks: + app_hooks.extend(extra_hooks) + + if not pecan_config: + pecan_config = get_pecan_config() + + if pecan_config.app.enable_acl: + app_hooks.append(hooks.AccessPolicyHook()) + + pecan.configuration.set_config(dict(pecan_config), overwrite=True) + + app = pecan.make_app( + pecan_config.app.root, + static_root=pecan_config.app.static_root, + template_path=pecan_config.app.template_path, + debug=False, + force_canonical=getattr(pecan_config.app, 'force_canonical', True), + hooks=app_hooks, + guess_content_type_from_ext=False, # Avoid mime-type lookup + ) + + # config_parser must contain the keystone_auth + if pecan_config.app.enable_acl: + CONF.import_group(acl.OPT_GROUP_NAME, acl.OPT_GROUP_PROVIDER) + return acl.install(app, CONF, pecan_config.app.acl_public_routes) + + return app + + +class VersionSelectorApplication(object): + def __init__(self): + pc = get_pecan_config() + pc.app.enable_acl = (CONF.auth_strategy == 'keystone') + self.v1 = setup_app(pecan_config=pc) + + def __call__(self, environ, start_response): + return self.v1(environ, start_response) diff --git a/software/software/authapi/auth_token.py b/software/software/authapi/auth_token.py new file mode 100755 index 00000000..71087cd4 --- /dev/null +++ b/software/software/authapi/auth_token.py @@ -0,0 +1,44 @@ +# -*- encoding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from keystonemiddleware import auth_token +from software import utils + + +class AuthTokenMiddleware(auth_token.AuthProtocol): + """A wrapper on Keystone auth_token middleware. + + Does not perform verification of authentication tokens + for public routes in the API. + + """ + def __init__(self, app, conf, public_api_routes=None): + if public_api_routes is None: + public_api_routes = [] + + self.public_api_routes = set(public_api_routes) + + super(AuthTokenMiddleware, self).__init__(app, conf) + + def __call__(self, env, start_response): + path = utils.safe_rstrip(env.get('PATH_INFO'), '/') + + if path in self.public_api_routes: + return self.app(env, start_response) # pylint: disable=no-member + + return super(AuthTokenMiddleware, self).__call__(env, start_response) # pylint: disable=too-many-function-args diff --git a/software/software/authapi/config.py b/software/software/authapi/config.py new file mode 100755 index 00000000..0126b827 --- /dev/null +++ b/software/software/authapi/config.py @@ -0,0 +1,23 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + +# Server Specific Configurations +server = { + 'port': '5497', + 'host': '0.0.0.0' +} + +# Pecan Application Configurations +app = { + 'root': 'software.api.controllers.root.RootController', + 'modules': ['software.api'], + 'static_root': '%(confdir)s/public', + 'template_path': '%(confdir)s/../templates', + 'debug': False, + 'enable_acl': True, + 'acl_public_routes': [], +} diff --git a/software/software/authapi/context.py b/software/software/authapi/context.py new file mode 100644 index 00000000..4759279f --- /dev/null +++ b/software/software/authapi/context.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from oslo_context import context + + +# Patching calls into fault. so only FM service type +# needs to be preserved in the service catalog +REQUIRED_SERVICE_TYPES = ('faultmanagement',) + + +class RequestContext(context.RequestContext): + """Extends security contexts from the OpenStack common library.""" + + def __init__(self, is_public_api=False, service_catalog=None, **kwargs): + """Stores several additional request parameters: + """ + super(RequestContext, self).__init__(**kwargs) + self.is_public_api = is_public_api + if service_catalog: + # Only include required parts of service_catalog + self.service_catalog = [s for s in service_catalog + if s.get('type') in REQUIRED_SERVICE_TYPES] + else: + # if list is empty or none + self.service_catalog = [] + + def to_dict(self): + value = super(RequestContext, self).to_dict() + value.update({'is_public_api': self.is_public_api, + 'project_name': self.project_name, + 'service_catalog': self.service_catalog}) + return value + + +def make_context(*args, **kwargs): + return RequestContext(*args, **kwargs) diff --git a/software/software/authapi/hooks.py b/software/software/authapi/hooks.py new file mode 100755 index 00000000..38370e4a --- /dev/null +++ b/software/software/authapi/hooks.py @@ -0,0 +1,134 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2012 New Dream Network, LLC (DreamHost) +# +# Author: Doug Hellmann # noqa: H105 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from oslo_config import cfg +from oslo_serialization import jsonutils +from pecan import hooks +from webob import exc + +from software.authapi.context import RequestContext +from software.authapi import policy +from software import utils + + +class ConfigHook(hooks.PecanHook): + """Attach the config object to the request so controllers can get to it.""" + + def before(self, state): + state.request.cfg = cfg.CONF + + +class ContextHook(hooks.PecanHook): + """Configures a request context and attaches it to the request. + + The following HTTP request headers are used: + + X-User-Id or X-User: + Used for context.user_id. + + X-Tenant-Id or X-Tenant: + Used for context.tenant. + + X-Auth-Token: + Used for context.auth_token. + + X-Roles: + Used for setting context.is_admin flag to either True or False. + The flag is set to True, if X-Roles contains either an administrator + or admin substring. Otherwise it is set to False. + + X-Project-Name: + Used for context.project_name. + + """ + def __init__(self, public_api_routes): + self.public_api_routes = public_api_routes + super(ContextHook, self).__init__() + + def before(self, state): + user_id = state.request.headers.get('X-User-Id') + user_id = state.request.headers.get('X-User', user_id) + tenant = state.request.headers.get('X-Tenant-Id') + tenant = state.request.headers.get('X-Tenant', tenant) + project_name = state.request.headers.get('X-Project-Name') + domain_id = state.request.headers.get('X-User-Domain-Id') + domain_name = state.request.headers.get('X-User-Domain-Name') + auth_token = state.request.headers.get('X-Auth-Token', None) + roles = state.request.headers.get('X-Roles', '').split(',') + catalog_header = state.request.headers.get('X-Service-Catalog') + service_catalog = None + if catalog_header: + try: + service_catalog = jsonutils.loads(catalog_header) + except ValueError: + raise exc.HTTPInternalServerError( + 'Invalid service catalog json.') + + credentials = { + 'project_name': project_name, + 'roles': roles + } + is_admin = policy.authorize('admin_in_system_projects', {}, + credentials, do_raise=False) + + path = utils.safe_rstrip(state.request.path, '/') + is_public_api = path in self.public_api_routes + + state.request.context = RequestContext( + auth_token=auth_token, + user=user_id, + tenant=tenant, + domain_id=domain_id, + domain_name=domain_name, + is_admin=is_admin, + is_public_api=is_public_api, + project_name=project_name, + roles=roles, + service_catalog=service_catalog) + + +class AccessPolicyHook(hooks.PecanHook): + """Verify that the user has the needed credentials + to execute the action. + """ + def before(self, state): + context = state.request.context + if not context.is_public_api: + controller = state.controller.__self__ + if hasattr(controller, 'enforce_policy'): + try: + controller_method = state.controller.__name__ + controller.enforce_policy(controller_method, state.request) + except Exception: + raise exc.HTTPForbidden() + else: + method = state.request.method + if method == 'GET': + has_api_access = policy.authorize( + 'reader_in_system_projects', {}, + context.to_dict(), do_raise=False) + else: + has_api_access = policy.authorize( + 'admin_in_system_projects', {}, + context.to_dict(), do_raise=False) + if not has_api_access: + raise exc.HTTPForbidden() diff --git a/software/software/authapi/policy.py b/software/software/authapi/policy.py new file mode 100755 index 00000000..6716e438 --- /dev/null +++ b/software/software/authapi/policy.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2011 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""Policy Engine For Unified Software Management.""" + +from oslo_config import cfg +from oslo_policy import policy + + +base_rules = [ + policy.RuleDefault('admin_in_system_projects', + 'role:admin and (project_name:admin or ' + + 'project_name:services)', + description='Admin user in system projects.'), + policy.RuleDefault('reader_in_system_projects', + 'role:reader and (project_name:admin or ' + + 'project_name:services)', + description='Reader user in system projects.'), + policy.RuleDefault('default', 'rule:admin_in_system_projects', + description='Default rule.'), +] + +CONF = cfg.CONF +_ENFORCER = None + + +def init(policy_file=None, rules=None, + default_rule=None, use_conf=True, overwrite=True): + """Init an Enforcer class. + + oslo policy supports change policy rule dynamically. + policy.enforce will reload the policy rules if it detects + the policy files have been touched. + + :param policy_file: Custom policy file to use, if none is + specified, ``conf.policy_file`` will be + used. + :param rules: Default dictionary / Rules to use. It will be + considered just in the first instantiation. If + :meth:`load_rules` with ``force_reload=True``, + :meth:`clear` or :meth:`set_rules` with + ``overwrite=True`` is called this will be overwritten. + :param default_rule: Default rule to use, conf.default_rule will + be used if none is specified. + :param use_conf: Whether to load rules from cache or config file. + :param overwrite: Whether to overwrite existing rules when reload rules + from config file. + """ + global _ENFORCER + if not _ENFORCER: + # https://docs.openstack.org/oslo.policy/latest/user/usage.html + _ENFORCER = policy.Enforcer(CONF, + policy_file=policy_file, + rules=rules, + default_rule=default_rule, + use_conf=use_conf, + overwrite=overwrite) + _ENFORCER.register_defaults(base_rules) + return _ENFORCER + + +def authorize(rule, target, creds, do_raise=True): + """A wrapper around 'authorize' from 'oslo_policy.policy'.""" + init() + return _ENFORCER.authorize(rule, target, creds, do_raise=do_raise) diff --git a/software/software/base.py b/software/software/base.py new file mode 100644 index 00000000..d5c2d44e --- /dev/null +++ b/software/software/base.py @@ -0,0 +1,171 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + +import socket +import struct +import subprocess +import sys +import time + +import software.utils as utils +import software.software_config as cfg +import software.constants as constants +from software.software_functions import LOG + + +class PatchService(object): + def __init__(self): + self.sock_out = None + self.sock_in = None + self.service_type = None + self.port = None + self.mcast_addr = None + self.socket_lock = None + + def update_config(self): + # Implemented in subclass + pass + + def socket_lock_acquire(self): + pass + + def socket_lock_release(self): + pass + + def setup_socket_ipv4(self): + mgmt_ip = cfg.get_mgmt_ip() + if mgmt_ip is None: + # Don't setup socket unless we have a mgmt ip + return None + + self.update_config() + + interface_addr = socket.inet_pton(socket.AF_INET, mgmt_ip) + + # Close sockets, if necessary + for s in [self.sock_out, self.sock_in]: + if s is not None: + s.close() + + self.sock_out = socket.socket(socket.AF_INET, + socket.SOCK_DGRAM) + self.sock_in = socket.socket(socket.AF_INET, + socket.SOCK_DGRAM) + + self.sock_out.setblocking(0) + self.sock_in.setblocking(0) + + self.sock_out.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.sock_in.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + self.sock_in.bind(('', self.port)) + + if self.mcast_addr: + # These options are for outgoing multicast messages + self.sock_out.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, interface_addr) + self.sock_out.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 1) + # Since only the controllers are sending to this address, + # we want the loopback so the local agent can receive it + self.sock_out.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1) + + # Register the multicast group + group = socket.inet_pton(socket.AF_INET, self.mcast_addr) + mreq = struct.pack('=4s4s', group, interface_addr) + + self.sock_in.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + + return self.sock_in + + def setup_socket_ipv6(self): + mgmt_ip = cfg.get_mgmt_ip() + if mgmt_ip is None: + # Don't setup socket unless we have a mgmt ip + return None + + self.update_config() + + # Close sockets, if necessary + for s in [self.sock_out, self.sock_in]: + if s is not None: + s.close() + + self.sock_out = socket.socket(socket.AF_INET6, + socket.SOCK_DGRAM) + self.sock_in = socket.socket(socket.AF_INET6, + socket.SOCK_DGRAM) + + self.sock_out.setblocking(0) + self.sock_in.setblocking(0) + + self.sock_out.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.sock_in.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + self.sock_out.bind((mgmt_ip, 0)) + self.sock_in.bind(('', self.port)) + + if self.mcast_addr: + # These options are for outgoing multicast messages + mgmt_ifindex = utils.if_nametoindex(cfg.get_mgmt_iface()) + self.sock_out.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, mgmt_ifindex) + self.sock_out.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, 1) + # Since only the controllers are sending to this address, + # we want the loopback so the local agent can receive it + self.sock_out.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_LOOP, 1) + + # Register the multicast group + if_index_packed = struct.pack('I', mgmt_ifindex) + group = socket.inet_pton(socket.AF_INET6, self.mcast_addr) + if_index_packed + self.sock_in.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, group) + + return self.sock_in + + def setup_socket(self): + self.socket_lock_acquire() + + try: + sock_in = None + if utils.get_management_version() == constants.ADDRESS_VERSION_IPV6: + sock_in = self.setup_socket_ipv6() + else: + sock_in = self.setup_socket_ipv4() + self.socket_lock_release() + return sock_in + except Exception: + LOG.exception("Failed to setup socket") + + # Close sockets, if necessary + for s in [self.sock_out, self.sock_in]: + if s is not None: + s.close() + + self.socket_lock_release() + + return None + + def audit_socket(self): + if not self.mcast_addr: + # Multicast address not configured, therefore nothing to do + return + + # Ensure multicast address is still allocated + cmd = "ip maddr show %s | awk 'BEGIN {ORS=\"\"}; {if ($2 == \"%s\") print $2}'" % \ + (cfg.get_mgmt_iface(), self.mcast_addr) + try: + result = subprocess.check_output(cmd, shell=True).decode(sys.stdout.encoding) + + if result == self.mcast_addr: + return + except subprocess.CalledProcessError as e: + LOG.error("Command output: %s", e.output) + return + + # Close the socket and set it up again + LOG.info("Detected missing multicast addr (%s). Reconfiguring", self.mcast_addr) + while self.setup_socket() is None: + LOG.info("Unable to setup sockets. Waiting to retry") + time.sleep(5) + LOG.info("Multicast address reconfigured") diff --git a/software/software/certificates.py b/software/software/certificates.py new file mode 100644 index 00000000..7c90303d --- /dev/null +++ b/software/software/certificates.py @@ -0,0 +1,51 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + +dev_certificate = b"""-----BEGIN CERTIFICATE----- + MIIDejCCAmKgAwIBAgICEAQwDQYJKoZIhvcNAQELBQAwQjELMAkGA1UEBhMCQ0Ex + EDAOBgNVBAgMB09udGFyaW8xITAfBgNVBAoMGFdpbmQgUml2ZXIgU3lzdGVtcywg + SW5jLjAeFw0xNzA4MTgxNDM3MjlaFw0yNzA4MTYxNDM3MjlaMEExCzAJBgNVBAYT + AkNBMRAwDgYDVQQIDAdPbnRhcmlvMSAwHgYDVQQKDBdXaW5kIFJpdmVyIFN5c3Rl + bXMsIEluYzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALcs0/Te6x69 + lxQOxudrF+uSC5F9r5bKUnZNWUKHyXKlN4SzZgWGs+fb/DqXIm7piuoQ6GH7GEQd + BEN1j/bwp30LZlv0Ur+8jhCvEdqsIP3vUXfv7pv0bomVs0Q8ZRI/FYZhjxYlyFKr + gZFV9WPP8S9SwfClHjaYRUudvwvjHHnnnkZ9blVFbXU0Xe83A8fWd0HNqAU1TlmK + 4CeSi4FI4aRKiXJnOvgv2UoJMI57rBIVKYRUH8uuFpPofOwjOM/Rd6r3Ir+4/CX6 + +/NALOBIEN6M05ZzoiyiH8NHELknQBqzNs0cXObJWpaSinAOcBnPCc7DNRwgQzjR + SdcE9FG1+LcCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0EHxYdT3Bl + blNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFDRbal2KxU0hQyv4 + MVnWrW96+aWoMB8GA1UdIwQYMBaAFJaLO1x8+jti7V6pLGbUyqpy0M36MA0GCSqG + SIb3DQEBCwUAA4IBAQBmcPFZzEoPtuMPCFvJ/0cmngp8yvCGxWz3JEDkdGYSCVGs + TG5e9DeltaHOk6yLvZSRY1so30GQnyB9q8v4DwEGVslKg8u9w/WEU81wl6Q2FZ5s + XRP6TASQ0Lbg9e4b3bnTITJJ8jT/zF29NaohgC2fg0UwVuldZLfa7FihJB4//OC1 + UdNEcmdqTVRqN2oco1n3ZUWKXvG2AvGsoiqu+lsWX1MXacoFvJexSACLrUvOoXMW + i38Ofp7XMCAm3rM0cXv7Uc9WCrgnTWbEvDgjGfRAmcM9moWGoWX6E46Xkojpkfle + Ss6CHAMK42aZ/+MWQlZEzNK49PtomGMjn5SuoK8u + -----END CERTIFICATE-----""" + +formal_certificate = b"""-----BEGIN CERTIFICATE----- + MIIDezCCAmOgAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwQjELMAkGA1UEBhMCQ0Ex + EDAOBgNVBAgMB09udGFyaW8xITAfBgNVBAoMGFdpbmQgUml2ZXIgU3lzdGVtcywg + SW5jLjAeFw0xNzA4MTgxNDM1MTJaFw0yNzA4MTYxNDM1MTJaMEIxCzAJBgNVBAYT + AkNBMRAwDgYDVQQIDAdPbnRhcmlvMSEwHwYDVQQKDBhXaW5kIFJpdmVyIFN5c3Rl + bXMsIEluYy4wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC+0fS8ybg8 + M37lW+lcR9LmQAR2zUJdbnl2L0fj3W/7W+PMm3mJWeQDTf19wf+qHHrgEkjxGp10 + BSXWZYdPyCdOjAay/Ew1s/waFeAQZpf4vv/9D1Y/4sVkqct9ibo5NVgvVsjqKVnX + IVhyzHlhBSUqYhZlS/SOx8JcLQWSUMJoP2XR4Tv28xIXi0Fuyp8QBwUmSwmvfPy4 + 0yxzfON/b8kHld5aTY353KLXh/5YWsn1zRlOYfS1OuJk4LGjm6HvmZtxPNUZk4vI + NA24rH4FKkuxyM3x8aPi3LE4G6GSrJDuNi28xzOj864rlFoyLODy/mov1YMR/g4k + d3mG6UbRckPxAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgENBB8WHU9w + ZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBTjyMN/AX07rEmB + 6sz6pnyt/m+eSzAfBgNVHSMEGDAWgBSWiztcfPo7Yu1eqSxm1MqqctDN+jANBgkq + hkiG9w0BAQsFAAOCAQEASpyCu/adGTvNjyy/tV+sL/kaVEKLA7q36HUrzQkTjMPX + y8L8PVZoeWprkz7cvYTyHmVTPLBvFkGEFVn8LWi9fTTp/UrHnxw6fvb+V78mOypi + 4A1aU9+dh3L6arpd4jZ4hDiLhEClesGCYVTVBdsrh3zSOc51nT4hosyBVpRd/VgQ + jhGJBBMEXASZceady4ajK5jnR3wF8oW/he4NYF97qh8WWKVsIYbwgLS0rT58q7qq + vpjPxMOahUdACkyPyt/XJICTlkanVD7KgG3oLWpc+3FWPHGr+F7mspPLZqUcEFDV + bGF+oDJ7p/tqHsNvPlRDVGqh0QdiAkKeS/SJC9jmAw== + -----END CERTIFICATE----- + """ diff --git a/software/software/cmd/api.py b/software/software/cmd/api.py index 20e2e40c..86975882 100644 --- a/software/software/cmd/api.py +++ b/software/software/cmd/api.py @@ -18,7 +18,7 @@ from software.api.app import setup_app LOG = logging.getLogger(__name__) # todo(abailey): these need to be part of config -API_PORT = 5490 +API_PORT = 5496 # Limit socket blocking to 5 seconds to allow for thread to shutdown API_SOCKET_TIMEOUT = 5.0 diff --git a/software/software/constants.py b/software/software/constants.py new file mode 100644 index 00000000..139e0718 --- /dev/null +++ b/software/software/constants.py @@ -0,0 +1,56 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + +import os +try: + # The tsconfig module is only available at runtime + import tsconfig.tsconfig as tsc + + INITIAL_CONFIG_COMPLETE_FLAG = os.path.join( + tsc.PLATFORM_CONF_PATH, ".initial_config_complete") +except Exception: + pass + +CLI_OPT_ALL = '--all' +CLI_OPT_DRY_RUN = '--dry-run' +CLI_OPT_RECURSIVE = '--recursive' +CLI_OPT_RELEASE = '--release' + +ADDRESS_VERSION_IPV4 = 4 +ADDRESS_VERSION_IPV6 = 6 +CONTROLLER_FLOATING_HOSTNAME = "controller" + +AVAILABLE = 'Available' +APPLIED = 'Applied' +PARTIAL_APPLY = 'Partial-Apply' +PARTIAL_REMOVE = 'Partial-Remove' +COMMITTED = 'Committed' +UNKNOWN = 'n/a' + +STATUS_OBSOLETE = 'OBS' +STATUS_RELEASED = 'REL' +STATUS_DEVELOPEMENT = 'DEV' + +PATCH_AGENT_STATE_IDLE = "idle" +PATCH_AGENT_STATE_INSTALLING = "installing" +PATCH_AGENT_STATE_INSTALL_FAILED = "install-failed" +PATCH_AGENT_STATE_INSTALL_REJECTED = "install-rejected" + +PATCH_STORAGE_DIR = "/opt/software" + +OSTREE_REF = "starlingx" +OSTREE_REMOTE = "debian" +FEED_OSTREE_BASE_DIR = "/var/www/pages/feed" +SYSROOT_OSTREE = "/sysroot/ostree/repo" +OSTREE_BASE_DEPLOYMENT_DIR = "/ostree/deploy/debian/deploy/" +PATCH_SCRIPTS_STAGING_DIR = "/var/www/pages/updates/software-scripts" + +LOOPBACK_INTERFACE_NAME = "lo" + +SEMANTIC_PREAPPLY = 'pre-apply' +SEMANTIC_PREREMOVE = 'pre-remove' +SEMANTIC_ACTIONS = [SEMANTIC_PREAPPLY, SEMANTIC_PREREMOVE] diff --git a/software/software/exceptions.py b/software/software/exceptions.py new file mode 100644 index 00000000..671459b5 --- /dev/null +++ b/software/software/exceptions.py @@ -0,0 +1,67 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + + +class PatchError(Exception): + """Base class for software exceptions.""" + + def __init__(self, message=None): + super(PatchError, self).__init__(message) + self.message = message + + def __str__(self): + return self.message or "" + + +class MetadataFail(PatchError): + """Metadata error.""" + pass + + +class ContentFail(PatchError): + """Content handling error.""" + pass + + +class OSTreeTarFail(PatchError): + """OSTree Tarball error.""" + pass + + +class OSTreeCommandFail(PatchError): + """OSTree Commands error.""" + pass + + +class SemanticFail(PatchError): + """Semantic check error.""" + pass + + +class RepoFail(PatchError): + """Repo error.""" + pass + + +class PatchFail(PatchError): + """General patching error.""" + pass + + +class PatchValidationFailure(PatchError): + """Patch validation error.""" + pass + + +class PatchMismatchFailure(PatchError): + """Patch validation error.""" + pass + + +class PatchInvalidRequest(PatchError): + """Invalid API request.""" + pass diff --git a/software/software/messages.py b/software/software/messages.py new file mode 100644 index 00000000..e44a35bd --- /dev/null +++ b/software/software/messages.py @@ -0,0 +1,66 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + +from software.software_functions import LOG + +PATCHMSG_UNKNOWN = 0 +PATCHMSG_HELLO = 1 +PATCHMSG_HELLO_ACK = 2 +PATCHMSG_SYNC_REQ = 3 +PATCHMSG_SYNC_COMPLETE = 4 +PATCHMSG_HELLO_AGENT = 5 +PATCHMSG_HELLO_AGENT_ACK = 6 +PATCHMSG_QUERY_DETAILED = 7 +PATCHMSG_QUERY_DETAILED_RESP = 8 +PATCHMSG_AGENT_INSTALL_REQ = 9 +PATCHMSG_AGENT_INSTALL_RESP = 10 +PATCHMSG_DROP_HOST_REQ = 11 +PATCHMSG_SEND_LATEST_FEED_COMMIT = 12 + +PATCHMSG_STR = { + PATCHMSG_UNKNOWN: "unknown", + PATCHMSG_HELLO: "hello", + PATCHMSG_HELLO_ACK: "hello-ack", + PATCHMSG_SYNC_REQ: "sync-req", + PATCHMSG_SYNC_COMPLETE: "sync-complete", + PATCHMSG_HELLO_AGENT: "hello-agent", + PATCHMSG_HELLO_AGENT_ACK: "hello-agent-ack", + PATCHMSG_QUERY_DETAILED: "query-detailed", + PATCHMSG_QUERY_DETAILED_RESP: "query-detailed-resp", + PATCHMSG_AGENT_INSTALL_REQ: "agent-install-req", + PATCHMSG_AGENT_INSTALL_RESP: "agent-install-resp", + PATCHMSG_DROP_HOST_REQ: "drop-host-req", + PATCHMSG_SEND_LATEST_FEED_COMMIT: "send-latest-feed-commit", +} + + +class PatchMessage(object): + def __init__(self, msgtype=PATCHMSG_UNKNOWN): + self.msgtype = msgtype + self.msgversion = 1 + self.message = {} + + def decode(self, data): + if 'msgtype' in data: + self.msgtype = data['msgtype'] + if 'msgversion' in data: + self.msgversion = data['msgversion'] + + def encode(self): + self.message['msgtype'] = self.msgtype + self.message['msgversion'] = self.msgversion + + def data(self): + return {'msgtype': self.msgtype} + + def msgtype_str(self): + if self.msgtype in PATCHMSG_STR: + return PATCHMSG_STR[self.msgtype] + return "invalid-type" + + def handle(self, sock, addr): # pylint: disable=unused-argument + LOG.info("Unhandled message type: %s", self.msgtype) diff --git a/software/software/ostree_utils.py b/software/software/ostree_utils.py new file mode 100644 index 00000000..3aa65674 --- /dev/null +++ b/software/software/ostree_utils.py @@ -0,0 +1,324 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" +import logging +import sh +import subprocess + +from software import constants +from software.exceptions import OSTreeCommandFail + +LOG = logging.getLogger('main_logger') + + +def get_ostree_latest_commit(ostree_ref, repo_path): + """ + Query ostree using ostree log --repo= + + :param ostree_ref: the ostree ref. + example: starlingx + :param repo_path: the path to the ostree repo: + example: /var/www/pages/feed/rel-22.06/ostree_repo + :return: The most recent commit of the repo + """ + + # Sample command and output that is parsed to get the commit + # + # Command: ostree log starlingx --repo=/var/www/pages/feed/rel-22.02/ostree_repo + # + # Output: + # + # commit 478bc21c1702b9b667b5a75fac62a3ef9203cc1767cbe95e89dface6dc7f205e + # ContentChecksum: 61fc5bb4398d73027595a4d839daeb404200d0899f6e7cdb24bb8fb6549912ba + # Date: 2022-04-28 18:58:57 +0000 + # + # Commit-id: starlingx-intel-x86-64-20220428185802 + # + # commit ad7057a94a1d06e38eaedee2ce3fe56826ae817497469bce5d5ac05bc506aaa7 + # ContentChecksum: dc42a42427a4f9e4de1210327c12b12ea3ad6a5d232497a903cc6478ca381e8b + # Date: 2022-04-28 18:05:43 +0000 + # + # Commit-id: starlingx-intel-x86-64-20220428180512 + + cmd = "ostree log %s --repo=%s" % (ostree_ref, repo_path) + try: + output = subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + info_msg = "OSTree log Error: return code: %s , Output: %s" \ + % (e.returncode, e.stderr.decode("utf-8")) + LOG.info(info_msg) + msg = "Failed to fetch ostree log for %s." % repo_path + raise OSTreeCommandFail(msg) + # Store the output of the above command in a string + output_string = output.stdout.decode('utf-8') + + # Parse the string to get the latest commit for the ostree + split_output_string = output_string.split() + latest_commit = split_output_string[1] + return latest_commit + + +def get_feed_latest_commit(patch_sw_version): + """ + Query ostree feed using ostree log --repo= + + :param patch_sw_version: software version for the feed + example: 22.06 + :return: The latest commit for the feed repo + """ + repo_path = "%s/rel-%s/ostree_repo" % (constants.FEED_OSTREE_BASE_DIR, + patch_sw_version) + return get_ostree_latest_commit(constants.OSTREE_REF, repo_path) + + +def get_sysroot_latest_commit(): + """ + Query ostree sysroot to determine the currently active commit + :return: The latest commit for sysroot repo + """ + return get_ostree_latest_commit(constants.OSTREE_REF, constants.SYSROOT_OSTREE) + + +def get_latest_deployment_commit(): + """ + Get the active deployment commit ID + :return: The commit ID associated with the active commit + """ + + # Sample command and output that is parsed to get the active commit + # associated with the deployment + # + # Command: ostree admin status + # + # Output: + # + # debian 0658a62854647b89caf5c0e9ed6ff62a6c98363ada13701d0395991569248d7e.0 (pending) + # origin refspec: starlingx + # * debian a5d8f8ca9bbafa85161083e9ca2259ff21e5392b7595a67f3bc7e7ab8cb583d9.0 + # Unlocked: hotfix + # origin refspec: starlingx + + cmd = "ostree admin status" + + try: + output = subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + msg = "Failed to fetch ostree admin status." + info_msg = "OSTree Admin Status Error: return code: %s , Output: %s" \ + % (e.returncode, e.stderr.decode("utf-8")) + LOG.info(info_msg) + raise OSTreeCommandFail(msg) + + # Store the output of the above command in a string + output_string = output.stdout.decode('utf-8') + + # Parse the string to get the active commit on this deployment + # Trim everything before * as * represents the active deployment commit + trimmed_output_string = output_string[output_string.index("*"):] + split_output_string = trimmed_output_string.split() + active_deployment_commit = split_output_string[2] + return active_deployment_commit + + +def update_repo_summary_file(repo_path): + """ + Updates the summary file for the specified ostree repo + :param repo_path: the path to the ostree repo: + example: /var/www/pages/feed/rel-22.06/ostree_repo + """ + cmd = "ostree summary --update --repo=%s" % repo_path + + try: + subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + msg = "Failed to update summary file for ostree repo %s." % (repo_path) + info_msg = "OSTree Summary Update Error: return code: %s , Output: %s" \ + % (e.returncode, e.stderr.decode("utf-8")) + LOG.info(info_msg) + raise OSTreeCommandFail(msg) + + +def reset_ostree_repo_head(commit, repo_path): + """ + Resets the ostree repo HEAD to the commit that is specified + :param commit: an existing commit on the ostree repo which we need the HEAD to point to + example: 478bc21c1702b9b667b5a75fac62a3ef9203cc1767cbe95e89dface6dc7f205e + :param repo_path: the path to the ostree repo: + example: /var/www/pages/feed/rel-22.06/ostree_repo + """ + cmd = "ostree reset %s %s --repo=%s" % (constants.OSTREE_REF, commit, repo_path) + try: + subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + msg = "Failed to reset head of ostree repo: %s to commit: %s" % (repo_path, commit) + info_msg = "OSTree Reset Error: return code: %s , Output: %s" \ + % (e.returncode, e.stderr.decode("utf-8")) + LOG.info(info_msg) + raise OSTreeCommandFail(msg) + + +def pull_ostree_from_remote(): + """ + Pull from remote ostree to sysroot ostree + """ + + cmd = "ostree pull %s --depth=-1" % constants.OSTREE_REMOTE + + try: + subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + msg = "Failed to pull from %s remote into sysroot ostree" % constants.OSTREE_REMOTE + info_msg = "OSTree Pull Error: return code: %s , Output: %s" \ + % (e.returncode, e.stderr.decode("utf-8")) + LOG.info(info_msg) + raise OSTreeCommandFail(msg) + + +def delete_ostree_repo_commit(commit, repo_path): + """ + Delete the specified commit from the ostree repo + :param commit: an existing commit on the ostree repo which we need to delete + example: 478bc21c1702b9b667b5a75fac62a3ef9203cc1767cbe95e89dface6dc7f205e + :param repo_path: the path to the ostree repo: + example: /var/www/pages/feed/rel-22.06/ostree_repo + """ + + cmd = "ostree prune --delete-commit %s --repo=%s" % (commit, repo_path) + try: + subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + msg = "Failed to delete commit %s from ostree repo %s" % (commit, repo_path) + info_msg = "OSTree Delete Commit Error: return code: %s , Output: %s" \ + % (e.returncode, e.stderr.decode("utf-8")) + LOG.info(info_msg) + raise OSTreeCommandFail(msg) + + +def create_deployment(): + """ + Create a new deployment while retaining the previous ones + """ + + cmd = "ostree admin deploy %s --no-prune --retain" % constants.OSTREE_REF + try: + subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + msg = "Failed to create an ostree deployment for sysroot ref %s." % constants.OSTREE_REF + info_msg = "OSTree Deployment Error: return code: %s , Output: %s" \ + % (e.returncode, e.stderr.decode("utf-8")) + LOG.info(info_msg) + raise OSTreeCommandFail(msg) + + +def fetch_pending_deployment(): + """ + Fetch the deployment ID of the pending deployment + :return: The deployment ID of the pending deployment + """ + + cmd = "ostree admin status | grep pending |awk '{printf $2}'" + + try: + output = subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + msg = "Failed to fetch ostree admin status." + info_msg = "OSTree Admin Status Error: return code: %s , Output: %s" \ + % (e.returncode, e.stderr.decode("utf-8")) + LOG.info(info_msg) + raise OSTreeCommandFail(msg) + + # Store the output of the above command in a string + pending_deployment = output.stdout.decode('utf-8') + + return pending_deployment + + +def mount_new_deployment(deployment_dir): + """ + Unmount /usr and /etc from the file system and remount it to directory + /usr and /etc respectively + :param deployment_dir: a path on the filesystem which points to the pending + deployment + example: /ostree/deploy/debian/deploy/ + """ + try: + new_usr_mount_dir = "%s/usr" % (deployment_dir) + new_etc_mount_dir = "%s/etc" % (deployment_dir) + sh.mount("--bind", "-o", "ro,noatime", new_usr_mount_dir, "/usr") + sh.mount("--bind", "-o", "rw,noatime", new_etc_mount_dir, "/etc") + except sh.ErrorReturnCode as e: + msg = "Failed to re-mount /usr and /etc." + info_msg = "OSTree Deployment Mount Error: Output: %s" \ + % (e.stderr.decode("utf-8")) + LOG.info(info_msg) + raise OSTreeCommandFail(msg) + finally: + try: + sh.mount("/usr/local/kubernetes/current/stage1") + sh.mount("/usr/local/kubernetes/current/stage2") + except sh.ErrorReturnCode: + msg = "Failed to mount kubernetes. Please manually run these commands:\n" \ + "sudo mount /usr/local/kubernetes/current/stage1\n" \ + "sudo mount /usr/local/kubernetes/current/stage2\n" + LOG.info(msg) + + +def delete_older_deployments(): + """ + Delete all older deployments after a reboot to save space + """ + # Sample command and output that is parsed to get the list of + # deployment IDs + # + # Command: ostree admin status | grep debian + # + # Output: + # + # * debian 3334dc80691a38c0ba6c519ec4b4b449f8420e98ac4d8bded3436ade56bb229d.2 + # debian 3334dc80691a38c0ba6c519ec4b4b449f8420e98ac4d8bded3436ade56bb229d.1 (rollback) + # debian 3334dc80691a38c0ba6c519ec4b4b449f8420e98ac4d8bded3436ade56bb229d.0 + + LOG.info("Inside delete_older_deployments of ostree_utils") + cmd = "ostree admin status | grep debian" + + try: + output = subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + msg = "Failed to fetch ostree admin status." + info_msg = "OSTree Admin Status Error: return code: %s , Output: %s" \ + % (e.returncode, e.stderr.decode("utf-8")) + LOG.info(info_msg) + raise OSTreeCommandFail(msg) + + # Store the output of the above command in a string + output_string = output.stdout.decode('utf-8') + + # Parse the string to get the latest commit for the ostree + split_output_string = output_string.split() + deployment_id_list = [] + for index, deployment_id in enumerate(split_output_string): + if deployment_id == "debian": + deployment_id_list.append(split_output_string[index + 1]) + + # After a reboot, the deployment ID at the 0th index of the list + # is always the active deployment and the deployment ID at the + # 1st index of the list is always the fallback deployment. + # We want to delete all deployments except the two mentioned above. + # This means we will undeploy all deployments starting from the + # 2nd index of deployment_id_list + + for index in reversed(range(2, len(deployment_id_list))): + try: + cmd = "ostree admin undeploy %s" % index + output = subprocess.run(cmd, shell=True, check=True, capture_output=True) + info_log = "Deleted ostree deployment %s" % deployment_id_list[index] + LOG.info(info_log) + except subprocess.CalledProcessError as e: + msg = "Failed to undeploy ostree deployment %s." % deployment_id_list[index] + info_msg = "OSTree Undeploy Error: return code: %s , Output: %s" \ + % (e.returncode, e.stderr.decode("utf-8")) + LOG.info(info_msg) + raise OSTreeCommandFail(msg) diff --git a/software/software/release_signing.py b/software/software/release_signing.py new file mode 100644 index 00000000..7d062373 --- /dev/null +++ b/software/software/release_signing.py @@ -0,0 +1,85 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + +import os +from Cryptodome.Signature import PKCS1_PSS +from Cryptodome.Hash import SHA256 +from software import release_verify + +# To save memory, read and hash 1M of files at a time +default_blocksize = 1 * 1024 * 1024 + +# When we sign patches, look for private keys in the following paths +# +# The (currently hardcoded) path on the signing server will be replaced +# by the capability to specify filename from calling function. +private_key_files = {release_verify.cert_type_formal_str: '/signing/keys/formal-private-key.pem', + release_verify.cert_type_dev_str: os.path.expandvars('$MY_REPO/build-tools/signing/dev-private-key.pem') + } + + +def sign_files(filenames, signature_file, private_key=None, cert_type=None): + """ + Utility function for signing data in files. + :param filenames: A list of files containing the data to be signed + :param signature_file: The name of the file to which the signature will be + stored + :param private_key: If specified, sign with this private key. Otherwise, + the files in private_key_files will be searched for + and used, if found. + :param cert_type: If specified, and private_key is not specified, sign + with a key of the specified type. e.g. 'dev' or 'formal' + """ + + # Hash the data across all files + blocksize = default_blocksize + data_hash = SHA256.new() + for filename in filenames: + with open(filename, 'rb') as infile: + data = infile.read(blocksize) + while len(data) > 0: + data_hash.update(data) + data = infile.read(blocksize) + + # Find a private key to use, if not already provided + need_resign_with_formal = False + if private_key is None: + if cert_type is not None: + # A Specific key is asked for + assert (cert_type in list(private_key_files)), "cert_type=%s is not a known cert type" % cert_type + dict_key = cert_type + filename = private_key_files[dict_key] + # print 'cert_type given: Checking to see if ' + filename + ' exists\n' + if not os.path.exists(filename) and dict_key == release_verify.cert_type_formal_str: + # The formal key is asked for, but is not locally available, + # substitute the dev key, and we will try to resign with the formal later. + dict_key = release_verify.cert_type_dev_str + filename = private_key_files[dict_key] + need_resign_with_formal = True + if os.path.exists(filename): + # print 'Getting private key from ' + filename + '\n' + private_key = release_verify.read_RSA_key(open(filename, 'rb').read()) + else: + # Search for available keys + for dict_key in private_key_files.keys(): + filename = private_key_files[dict_key] + # print 'Search for available keys: Checking to see if ' + filename + ' exists\n' + if os.path.exists(filename): + # print 'Getting private key from ' + filename + '\n' + private_key = release_verify.read_RSA_key(open(filename, 'rb').read()) + + assert (private_key is not None), "Could not find signing key" + + # Encrypt the hash (sign the data) with the key we find + signer = PKCS1_PSS.new(private_key) + signature = signer.sign(data_hash) + + # Save it + with open(signature_file, 'wb') as outfile: + outfile.write(signature) + + return need_resign_with_formal diff --git a/software/software/release_verify.py b/software/software/release_verify.py new file mode 100644 index 00000000..ae49fd0d --- /dev/null +++ b/software/software/release_verify.py @@ -0,0 +1,191 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + +import os +import logging + +from Cryptodome.Signature import PKCS1_v1_5 +from Cryptodome.Signature import PKCS1_PSS +from Cryptodome.Hash import SHA256 +from Cryptodome.PublicKey import RSA +from Cryptodome.Util.asn1 import DerSequence +from binascii import a2b_base64 + +from software.certificates import dev_certificate +from software.certificates import formal_certificate + +# To save memory, read and hash 1M of files at a time +default_blocksize = 1 * 1024 * 1024 + +dev_certificate_marker = '/etc/pki/wrs/dev_certificate_enable.bin' +DEV_CERT_CONTENTS = b'Titanium patching' +LOG = logging.getLogger('main_logger') + +cert_type_dev_str = 'dev' +cert_type_formal_str = 'formal' +cert_type_dev = [cert_type_dev_str] +cert_type_formal = [cert_type_formal_str] +cert_type_all = [cert_type_dev_str, cert_type_formal_str] + + +def verify_hash(data_hash, signature_bytes, certificate_list): + """ + Checks that a hash's signature can be validated against an approved + certificate + :param data_hash: A hash of the data to be validated + :param signature_bytes: A pre-generated signature (typically, the hash + encrypted with a private key) + :param certificate_list: A list of approved certificates or public keys + which the signature is validated against + :return: True if the signature was validated against a certificate + """ + verified = False + for cert in certificate_list: + if verified: + break + pub_key = read_RSA_key(cert) + pub_key.exportKey() + + # PSS is the recommended signature scheme, but some tools (like OpenSSL) + # use the older v1_5 scheme. We try to validate against both. + # + # We use PSS for patch validation, but use v1_5 for ISO validation + # since we want to generate detached sigs that a customer can validate + # OpenSSL + verifier = PKCS1_PSS.new(pub_key) + try: + verified = verifier.verify(data_hash, signature_bytes) # pylint: disable=not-callable + except ValueError: + verified = False + + if not verified: + verifier = PKCS1_v1_5.new(pub_key) + try: + verified = verifier.verify(data_hash, signature_bytes) # pylint: disable=not-callable + except ValueError: + verified = False + + return verified + + +def get_public_certificates_by_type(cert_type=None): + """ + Builds a list of accepted certificates which can be used to validate + further things. This list may contain multiple certificates depending on + the configuration of the system and the value of cert_type. + + :param cert_type: A list of strings, certificate types to include in list + 'formal' - include formal certificate if available + 'dev' - include developer certificate if available + :return: A list of certificates in PEM format + """ + + if cert_type is None: + cert_type = cert_type_all + + cert_list = [] + + if cert_type_formal_str in cert_type: + cert_list.append(formal_certificate) + + if cert_type_dev_str in cert_type: + cert_list.append(dev_certificate) + + return cert_list + + +def get_public_certificates(): + """ + Builds a list of accepted certificates which can be used to validate + further things. This list may contain multiple certificates depending on + the configuration of the system (for instance, should we include the + developer certificate in the list). + :return: A list of certificates in PEM format + """ + cert_list = [formal_certificate] + + # We enable the dev certificate based on the presence of a file. This file + # contains a hash of an arbitrary string ('Titanum patching') which has been + # encrypted with our formal private key. If the file is present (and valid) + # then we add the developer key to the approved certificates list + if os.path.exists(dev_certificate_marker): + with open(dev_certificate_marker, 'rb') as infile: + signature = infile.read() + data_hash = SHA256.new(DEV_CERT_CONTENTS) + if verify_hash(data_hash, signature, cert_list): + cert_list.append(dev_certificate) + else: + msg = "Invalid data found in " + dev_certificate_marker + LOG.error(msg) + + return cert_list + + +def read_RSA_key(key_data): + """ + Utility function for reading an RSA key half from encoded data + :param key_data: PEM data containing raw key or X.509 certificate + :return: An RSA key object + """ + try: + # Handle data that is just a raw key + key = RSA.importKey(key_data) + except ValueError: + # The RSA.importKey function cannot read X.509 certificates directly + # (depending on the version of the Crypto library). Instead, we + # may need to extract the key from the certificate before building + # the key object + # + # We need to strip the BEGIN and END lines from PEM first + x509lines = key_data.replace(' ', '').split() + x509text = ''.join(x509lines[1:-1]) + x509data = DerSequence() + x509data.decode(a2b_base64(x509text)) + + # X.509 contains a few parts. The first part (index 0) is the + # certificate itself, (TBS or "to be signed" cert) and the 7th field + # of that cert is subjectPublicKeyInfo, which can be imported. + # RFC3280 + tbsCert = DerSequence() + tbsCert.decode(x509data[0]) + + # Initialize RSA key from the subjectPublicKeyInfo field + key = RSA.importKey(tbsCert[6]) + return key + + +def verify_files(filenames, signature_file, cert_type=None): + """ + Verify data files against a detached signature. + :param filenames: A list of files containing the data which was signed + :param public_key_file: A file containing the public key or certificate + corresponding to the key which signed the data + :param signature_file: The name of the file containing the signature + :param cert_type: Only use specified certififcate type to verify (dev/formal) + :return: True if the signature was verified, False otherwise + """ + + # Hash the data across all files + blocksize = default_blocksize + data_hash = SHA256.new() + for filename in filenames: + with open(filename, 'rb') as infile: + data = infile.read(blocksize) + while len(data) > 0: + data_hash.update(data) + data = infile.read(blocksize) + + # Get the signature + with open(signature_file, 'rb') as sig_file: + signature_bytes = sig_file.read() + + # Verify the signature + if cert_type is None: + certificate_list = get_public_certificates() + else: + certificate_list = get_public_certificates_by_type(cert_type=cert_type) + return verify_hash(data_hash, signature_bytes, certificate_list) diff --git a/software/software/software_agent.py b/software/software/software_agent.py new file mode 100644 index 00000000..e5ca846d --- /dev/null +++ b/software/software/software_agent.py @@ -0,0 +1,744 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" +import json +import os +import random +import requests +import select +import shutil +import socket +import subprocess +import sys +import time + +from software import ostree_utils +from software.software_functions import configure_logging +from software.software_functions import LOG +import software.software_config as cfg +from software.base import PatchService +from software.exceptions import OSTreeCommandFail +import software.utils as utils +import software.messages as messages +import software.constants as constants + +from tsconfig.tsconfig import http_port +from tsconfig.tsconfig import install_uuid +from tsconfig.tsconfig import subfunctions +from tsconfig.tsconfig import SW_VERSION + +pidfile_path = "/var/run/software_agent.pid" +agent_running_after_reboot_flag = \ + "/var/run/software_agent_running_after_reboot" +node_is_patched_file = "/var/run/node_is_patched" +node_is_patched_rr_file = "/var/run/node_is_patched_rr" +patch_installing_file = "/var/run/patch_installing" +patch_failed_file = "/var/run/patch_install_failed" +node_is_locked_file = "/var/run/.node_locked" +ostree_pull_completed_deployment_pending_file = \ + "/var/run/ostree_pull_completed_deployment_pending" +mount_pending_file = "/var/run/mount_pending" +insvc_patch_scripts = "/run/software/software-scripts" +insvc_patch_flags = "/run/software/software-flags" +insvc_patch_restart_agent = "/run/software/.restart.software-agent" + +run_insvc_patch_scripts_cmd = "/usr/sbin/run-software-scripts" + +pa = None + +http_port_real = http_port + + +def setflag(fname): + try: + with open(fname, "w") as f: + f.write("%d\n" % os.getpid()) + except Exception: + LOG.exception("Failed to update %s flag", fname) + + +def clearflag(fname): + if os.path.exists(fname): + try: + os.remove(fname) + except Exception: + LOG.exception("Failed to clear %s flag", fname) + + +def pull_restart_scripts_from_controller(): + # If the rsync fails, it raises an exception to + # the caller "handle_install()" and fails the + # host-install request for this host + output = subprocess.check_output(["rsync", + "-acv", + "--delete", + "--exclude", "tmp", + "rsync://controller/repo/patch-scripts/", + "%s/" % insvc_patch_scripts], + stderr=subprocess.STDOUT) + LOG.info("Synced restart scripts from controller: %s", output) + + +def check_install_uuid(): + controller_install_uuid_url = "http://controller:%s/feed/rel-%s/install_uuid" % (http_port_real, SW_VERSION) + try: + req = requests.get(controller_install_uuid_url) + if req.status_code != 200: + # If we're on controller-1, controller-0 may not have the install_uuid + # matching this release, if we're in an upgrade. If the file doesn't exist, + # bypass this check + if socket.gethostname() == "controller-1": + return True + + LOG.error("Failed to get install_uuid from controller") + return False + except requests.ConnectionError: + LOG.error("Failed to connect to controller") + return False + + controller_install_uuid = str(req.text).rstrip() + + if install_uuid != controller_install_uuid: + LOG.error("Local install_uuid=%s doesn't match controller=%s", install_uuid, controller_install_uuid) + return False + + return True + + +class PatchMessageSendLatestFeedCommit(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_SEND_LATEST_FEED_COMMIT) + + def decode(self, data): + global pa + messages.PatchMessage.decode(self, data) + if 'latest_feed_commit' in data: + pa.latest_feed_commit = data['latest_feed_commit'] + + def encode(self): + messages.PatchMessage.encode(self) + + def handle(self, sock, addr): + global pa + # Check if the node is patch current + pa.query() + + +class PatchMessageHelloAgent(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_HELLO_AGENT) + self.patch_op_counter = 0 + + def decode(self, data): + messages.PatchMessage.decode(self, data) + if 'patch_op_counter' in data: + self.patch_op_counter = data['patch_op_counter'] + + def encode(self): + messages.PatchMessage.encode(self) + + def handle(self, sock, addr): + # Send response + + # + # If a user tries to do a host-install on an unlocked node, + # without bypassing the lock check (either via in-service + # patch or --force option), the agent will set its state + # to Install-Rejected in order to report back the rejection. + # However, since this should just be a transient state, + # we don't want the client reporting the Install-Rejected + # state indefinitely, so reset it to Idle after a minute or so. + # + if pa.state == constants.PATCH_AGENT_STATE_INSTALL_REJECTED: + if os.path.exists(node_is_locked_file): + # Node has been locked since rejected attempt. Reset the state + pa.state = constants.PATCH_AGENT_STATE_IDLE + elif (time.time() - pa.rejection_timestamp) > 60: + # Rejected state for more than a minute. Reset it. + pa.state = constants.PATCH_AGENT_STATE_IDLE + + if self.patch_op_counter > 0: + pa.handle_patch_op_counter(self.patch_op_counter) + + resp = PatchMessageHelloAgentAck() + resp.send(sock) + + def send(self, sock): # pylint: disable=unused-argument + LOG.error("Should not get here") + + +class PatchMessageHelloAgentAck(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_HELLO_AGENT_ACK) + + def encode(self): + global pa + messages.PatchMessage.encode(self) + self.message['query_id'] = pa.query_id + self.message['out_of_date'] = pa.changes + self.message['hostname'] = socket.gethostname() + self.message['requires_reboot'] = pa.node_is_patched + self.message['patch_failed'] = pa.patch_failed + self.message['sw_version'] = SW_VERSION + self.message['state'] = pa.state + + def handle(self, sock, addr): + LOG.error("Should not get here") + + def send(self, sock): + global pa + self.encode() + message = json.dumps(self.message) + sock.sendto(str.encode(message), (pa.controller_address, cfg.controller_port)) + + +class PatchMessageQueryDetailed(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_QUERY_DETAILED) + + def decode(self, data): + messages.PatchMessage.decode(self, data) + + def encode(self): + # Nothing to add to the HELLO_AGENT, so just call the super class + messages.PatchMessage.encode(self) + + def handle(self, sock, addr): + # Send response + LOG.info("Handling detailed query") + resp = PatchMessageQueryDetailedResp() + resp.send(sock) + + def send(self, sock): # pylint: disable=unused-argument + LOG.error("Should not get here") + + +class PatchMessageQueryDetailedResp(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_QUERY_DETAILED_RESP) + + def encode(self): + global pa + messages.PatchMessage.encode(self) + self.message['latest_sysroot_commit'] = pa.latest_sysroot_commit + self.message['nodetype'] = cfg.nodetype + self.message['sw_version'] = SW_VERSION + self.message['subfunctions'] = subfunctions + self.message['state'] = pa.state + + def handle(self, sock, addr): + LOG.error("Should not get here") + + def send(self, sock): + self.encode() + message = json.dumps(self.message) + sock.sendall(str.encode(message)) + + +class PatchMessageAgentInstallReq(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_AGENT_INSTALL_REQ) + self.force = False + + def decode(self, data): + messages.PatchMessage.decode(self, data) + if 'force' in data: + self.force = data['force'] + + def encode(self): + # Nothing to add to the HELLO_AGENT, so just call the super class + messages.PatchMessage.encode(self) + + def handle(self, sock, addr): + LOG.info("Handling host install request, force=%s", self.force) + global pa + resp = PatchMessageAgentInstallResp() + + if not self.force: + setflag(node_is_patched_rr_file) + + if not os.path.exists(node_is_locked_file): + if self.force: + LOG.info("Installing on unlocked node, with force option") + else: + LOG.info("Rejecting install request on unlocked node") + pa.state = constants.PATCH_AGENT_STATE_INSTALL_REJECTED + pa.rejection_timestamp = time.time() + resp.status = False + resp.reject_reason = 'Node must be locked.' + resp.send(sock, addr) + return + resp.status = pa.handle_install() + resp.send(sock, addr) + + def send(self, sock): # pylint: disable=unused-argument + LOG.error("Should not get here") + + +class PatchMessageAgentInstallResp(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_AGENT_INSTALL_RESP) + self.status = False + self.reject_reason = None + + def encode(self): + global pa + messages.PatchMessage.encode(self) + self.message['status'] = self.status + if self.reject_reason is not None: + self.message['reject_reason'] = self.reject_reason + + def handle(self, sock, addr): + LOG.error("Should not get here") + + def send(self, sock, addr): + address = (addr[0], cfg.controller_port) + self.encode() + message = json.dumps(self.message) + sock.sendto(str.encode(message), address) + + # Send a hello ack to follow it + resp = PatchMessageHelloAgentAck() + resp.send(sock) + + +class PatchAgent(PatchService): + def __init__(self): + PatchService.__init__(self) + self.sock_out = None + self.sock_in = None + self.controller_address = None + self.listener = None + self.changes = False + self.latest_feed_commit = None + self.latest_sysroot_commit = None + self.patch_op_counter = 0 + self.node_is_patched = os.path.exists(node_is_patched_file) + self.node_is_patched_timestamp = 0 + self.query_id = 0 + self.state = constants.PATCH_AGENT_STATE_IDLE + self.last_config_audit = 0 + self.rejection_timestamp = 0 + self.last_repo_revision = None + + # Check state flags + if os.path.exists(patch_installing_file): + # We restarted while installing. Change to failed + setflag(patch_failed_file) + os.remove(patch_installing_file) + + if os.path.exists(patch_failed_file): + self.state = constants.PATCH_AGENT_STATE_INSTALL_FAILED + + self.patch_failed = os.path.exists(patch_failed_file) + + def update_config(self): + cfg.read_config() + + if self.port != cfg.agent_port: + self.port = cfg.agent_port + + # Loopback interface does not support multicast messaging, therefore + # revert to using unicast messaging when configured against the + # loopback device + if cfg.get_mgmt_iface() == constants.LOOPBACK_INTERFACE_NAME: + self.mcast_addr = None + self.controller_address = cfg.get_mgmt_ip() + else: + self.mcast_addr = cfg.agent_mcast_group + self.controller_address = cfg.controller_mcast_group + + def setup_tcp_socket(self): + address_family = utils.get_management_family() + self.listener = socket.socket(address_family, socket.SOCK_STREAM) + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.listener.bind(('', self.port)) + self.listener.listen(2) # Allow two connections, for two controllers + + def query(self): + """Check current patch state """ + if not check_install_uuid(): + LOG.info("Failed install_uuid check. Skipping query") + return False + + # Generate a unique query id + self.query_id = random.random() + + # determine OSTREE state of the system and the patches + self.changes = False + + active_sysroot_commit = ostree_utils.get_sysroot_latest_commit() + self.latest_sysroot_commit = active_sysroot_commit + self.last_repo_revision = active_sysroot_commit + + # latest_feed_commit is sent from patch controller + # if unprovisioned (no mgmt ip) attempt to query it + if self.latest_feed_commit is None: + if self.sock_out is None: + try: + self.latest_feed_commit = ostree_utils.get_feed_latest_commit(SW_VERSION) + except OSTreeCommandFail: + LOG.warning("Unable to query latest feed commit") + # latest_feed_commit will remain as None + + if self.latest_feed_commit: + if active_sysroot_commit != self.latest_feed_commit: + LOG.info("Active Sysroot Commit:%s does not match " + "active controller's Feed Repo Commit: %s", + active_sysroot_commit, self.latest_feed_commit) + self.changes = True + + return True + + def handle_install(self, + verbose_to_stdout=False, + disallow_insvc_patch=False, + delete_older_deployments=False): + # + # The disallow_insvc_patch parameter is set when we're installing + # the patch during init. At that time, we don't want to deal with + # in-service patch scripts, so instead we'll treat any patch as + # a reboot-required when this parameter is set. Rather than running + # any scripts, the RR flag will be set, which will result in the node + # being rebooted immediately upon completion of the installation. + # + # The delete_older_deployments is set when the system has + # been rebooted. + # + + LOG.info("Handling install") + + # Check the INSTALL_UUID first. If it doesn't match the active + # controller, we don't want to install patches. + if not check_install_uuid(): + LOG.error("Failed install_uuid check. Skipping install") + + self.patch_failed = True + setflag(patch_failed_file) + self.state = constants.PATCH_AGENT_STATE_INSTALL_FAILED + + # Send a hello to provide a state update + if self.sock_out is not None: + hello_ack = PatchMessageHelloAgentAck() + hello_ack.send(self.sock_out) + + return False + + self.state = constants.PATCH_AGENT_STATE_INSTALLING + setflag(patch_installing_file) + + if delete_older_deployments: + ostree_utils.delete_older_deployments() + + try: + # Create insvc patch directories + if not os.path.exists(insvc_patch_scripts): + os.makedirs(insvc_patch_scripts, 0o700) + if not os.path.exists(insvc_patch_flags): + os.makedirs(insvc_patch_flags, 0o700) + except Exception: + LOG.exception("Failed to create in-service patch directories") + + # Send a hello to provide a state update + if self.sock_out is not None: + hello_ack = PatchMessageHelloAgentAck() + hello_ack.send(self.sock_out) + + # Build up the install set + if verbose_to_stdout: + print("Checking for software updates...") + self.query() # sets self.changes + + changed = False + success = True + + if self.changes or \ + os.path.exists(ostree_pull_completed_deployment_pending_file) or \ + os.path.exists(mount_pending_file): + try: + # Pull changes from remote to the sysroot ostree + # The remote value is configured inside + # "/sysroot/ostree/repo/config" file + ostree_utils.pull_ostree_from_remote() + setflag(ostree_pull_completed_deployment_pending_file) + except OSTreeCommandFail: + LOG.exception("Failed to pull changes and create deployment" + "during host-install.") + success = False + + try: + # Create a new deployment once the changes are pulled + ostree_utils.create_deployment() + + changed = True + clearflag(ostree_pull_completed_deployment_pending_file) + + except OSTreeCommandFail: + LOG.exception("Failed to pull changes and create deployment" + "during host-install.") + success = False + + if changed: + # Update the node_is_patched flag + setflag(node_is_patched_file) + + self.node_is_patched = True + if verbose_to_stdout: + print("This node has been patched.") + + if os.path.exists(node_is_patched_rr_file): + LOG.info("Reboot is required. Skipping patch-scripts") + elif disallow_insvc_patch: + LOG.info("Disallowing patch-scripts. Treating as reboot-required") + setflag(node_is_patched_rr_file) + else: + LOG.info("Mounting the new deployment") + try: + pending_deployment = ostree_utils.fetch_pending_deployment() + deployment_dir = constants.OSTREE_BASE_DEPLOYMENT_DIR + pending_deployment + setflag(mount_pending_file) + ostree_utils.mount_new_deployment(deployment_dir) + clearflag(mount_pending_file) + LOG.info("Running in-service patch-scripts") + pull_restart_scripts_from_controller() + subprocess.check_output(run_insvc_patch_scripts_cmd, stderr=subprocess.STDOUT) + + # Clear the node_is_patched flag, since we've handled it in-service + clearflag(node_is_patched_file) + self.node_is_patched = False + except subprocess.CalledProcessError as e: + LOG.exception("In-Service patch installation failed") + LOG.error("Command output: %s", e.output) + success = False + + # Clear the in-service patch dirs + if os.path.exists(insvc_patch_scripts): + shutil.rmtree(insvc_patch_scripts, ignore_errors=True) + if os.path.exists(insvc_patch_flags): + shutil.rmtree(insvc_patch_flags, ignore_errors=True) + + if success: + self.patch_failed = False + clearflag(patch_failed_file) + self.state = constants.PATCH_AGENT_STATE_IDLE + else: + # Update the patch_failed flag + self.patch_failed = True + setflag(patch_failed_file) + self.state = constants.PATCH_AGENT_STATE_INSTALL_FAILED + + clearflag(patch_installing_file) + self.query() + + if self.changes: + LOG.warning("Installing the patch did not change the patch current status") + + # Send a hello to provide a state update + if self.sock_out is not None: + hello_ack = PatchMessageHelloAgentAck() + hello_ack.send(self.sock_out) + + # Indicate if the method was successful + # success means no change needed, or a change worked. + return success + + def handle_patch_op_counter(self, counter): + changed = False + if os.path.exists(node_is_patched_file): + # The node has been patched. Run a query if: + # - node_is_patched didn't exist previously + # - node_is_patched timestamp changed + timestamp = os.path.getmtime(node_is_patched_file) + if not self.node_is_patched: + self.node_is_patched = True + self.node_is_patched_timestamp = timestamp + changed = True + elif self.node_is_patched_timestamp != timestamp: + self.node_is_patched_timestamp = timestamp + changed = True + elif self.node_is_patched: + self.node_is_patched = False + self.node_is_patched_timestamp = 0 + changed = True + + if self.patch_op_counter < counter: + self.patch_op_counter = counter + changed = True + + if changed: + rc = self.query() + if not rc: + # Query failed. Reset the op counter + self.patch_op_counter = 0 + + def run(self): + self.setup_socket() + + while self.sock_out is None: + # Check every thirty seconds? + # Once we've got a conf file, tied into packstack, + # we'll get restarted when the file is updated, + # and this should be unnecessary. + time.sleep(30) + self.setup_socket() + + self.setup_tcp_socket() + + # Ok, now we've got our socket. + # Let's let the controllers know we're here + hello_ack = PatchMessageHelloAgentAck() + hello_ack.send(self.sock_out) + + first_hello = True + + connections = [] + + timeout = time.time() + 30.0 + remaining = 30 + + while True: + inputs = [self.sock_in, self.listener] + connections + outputs = [] + + rlist, wlist, xlist = select.select(inputs, outputs, inputs, remaining) + + remaining = int(timeout - time.time()) + if remaining <= 0 or remaining > 30: + timeout = time.time() + 30.0 + remaining = 30 + + if (len(rlist) == 0 and + len(wlist) == 0 and + len(xlist) == 0): + # Timeout hit + self.audit_socket() + continue + + for s in rlist: + if s == self.listener: + conn, addr = s.accept() + connections.append(conn) + continue + + data = '' + addr = None + msg = None + + if s == self.sock_in: + # Receive from UDP + data, addr = s.recvfrom(1024) + else: + # Receive from TCP + while True: + try: + packet = s.recv(1024) + except socket.error: + LOG.exception("Socket error on recv") + data = '' + break + + if packet: + data += packet.decode() + + if data == '': + break + + try: + json.loads(data) + break + except ValueError: + # Message is incomplete + continue + else: + # End of TCP message received + break + + if data == '': + # Connection dropped + connections.remove(s) + s.close() + continue + + msgdata = json.loads(data) + + # For now, discard any messages that are not msgversion==1 + if 'msgversion' in msgdata and msgdata['msgversion'] != 1: + continue + + if 'msgtype' in msgdata: + if msgdata['msgtype'] == messages.PATCHMSG_HELLO_AGENT: + if first_hello: + self.query() + first_hello = False + + msg = PatchMessageHelloAgent() + elif msgdata['msgtype'] == messages.PATCHMSG_QUERY_DETAILED: + msg = PatchMessageQueryDetailed() + elif msgdata['msgtype'] == messages.PATCHMSG_SEND_LATEST_FEED_COMMIT: + msg = PatchMessageSendLatestFeedCommit() + elif msgdata['msgtype'] == messages.PATCHMSG_AGENT_INSTALL_REQ: + msg = PatchMessageAgentInstallReq() + + if msg is None: + msg = messages.PatchMessage() + + msg.decode(msgdata) + if s == self.sock_in: + msg.handle(self.sock_out, addr) + else: + msg.handle(s, addr) + + for s in xlist: + if s in connections: + connections.remove(s) + s.close() + + # Check for in-service patch restart flag + if os.path.exists(insvc_patch_restart_agent): + # Make sure it's safe to restart, ie. no reqs queued + rlist, wlist, xlist = select.select(inputs, outputs, inputs, 0) + if (len(rlist) == 0 and + len(wlist) == 0 and + len(xlist) == 0): + # Restart + LOG.info("In-service patch restart flag detected. Exiting.") + os.remove(insvc_patch_restart_agent) + exit(0) + + +def main(): + global pa + + configure_logging() + + cfg.read_config() + + pa = PatchAgent() + pa.query() + if os.path.exists(agent_running_after_reboot_flag): + delete_older_deployments_flag = False + else: + setflag(agent_running_after_reboot_flag) + delete_older_deployments_flag = True + + if len(sys.argv) <= 1: + pa.run() + elif sys.argv[1] == "--install": + if not check_install_uuid(): + # In certain cases, the lighttpd server could still be running using + # its default port 80, as opposed to the port configured in platform.conf + global http_port_real + LOG.info("Failed install_uuid check via http_port=%s. Trying with default port 80", http_port_real) + http_port_real = 80 + + pa.handle_install(verbose_to_stdout=True, + disallow_insvc_patch=True, + delete_older_deployments=delete_older_deployments_flag) + elif sys.argv[1] == "--status": + rc = 0 + if pa.changes: + rc = 1 + exit(rc) diff --git a/software/software/software_client.py b/software/software/software_client.py new file mode 100644 index 00000000..da83f9fe --- /dev/null +++ b/software/software/software_client.py @@ -0,0 +1,1449 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" +import json +import os +import re +import requests +import signal +import subprocess +import sys +import textwrap +import time + +from requests_toolbelt import MultipartEncoder + +import software.constants as constants + +from tsconfig.tsconfig import SW_VERSION as RUNNING_SW_VERSION + +api_addr = "127.0.0.1:5493" +auth_token = None + +TERM_WIDTH = 72 +VIRTUAL_REGION = 'SystemController' +IPV6_FAMILY = 6 + + +help_upload = "Upload one or more software release versions to the system." +help_upload_dir = "Upload software release versions from one or more directories to the system." +help_apply = "Apply one or more patches. This adds the specified patches " + \ + "to the repository, making the update(s) available to the " + \ + "hosts in the system. Use --all to apply all available patches." +help_remove = "Remove one or more patches. This removes the specified " + \ + "patches from the repository." +help_delete = "Delete one or more software release versions from the system." +help_query = "Query system patches. Optionally, specify 'query applied' " + \ + "to query only those patches that are applied, or 'query available' " + \ + "to query those that are not." +help_show = "Show details for specified patches." +help_what_requires = "List patches that require the specified patches." +help_query_hosts = "Query patch states for hosts in the system." +help_host_install = "Trigger patch install/remove on specified host. " + \ + "To force install on unlocked node, use the --force option." +help_host_install_async = "Trigger patch install/remove on specified host. " + \ + "To force install on unlocked node, use the --force option." + \ + " Note: This command returns immediately upon dispatching installation request." +help_patch_args = "Patches are specified as a space-separated list of patch IDs." +help_install_local = "Trigger patch install/remove on the local host. " + \ + "This command can only be used for patch installation prior to initial " + \ + "configuration." +help_drop_host = "Drop specified host from table." +help_query_dependencies = "List dependencies for specified patch. Use " + \ + constants.CLI_OPT_RECURSIVE + " for recursive query." +help_is_applied = "Query Applied state for list of patches. " + \ + "Returns True if all are Applied, False otherwise." +help_is_available = "Query Available state for list of patches. " + \ + "Returns True if all are Available, False otherwise." +help_report_app_dependencies = "Report application patch dependencies, " + \ + "specifying application name with --app option, plus a list of patches. " + \ + "Reported dependencies can be dropped by specifying app with no patch list." +help_query_app_dependencies = "Display set of reported application patch " + \ + "dependencies." +help_commit = "Commit patches to free disk space. WARNING: This action " + \ + "is irreversible!" +help_region_name = "Send the request to a specified region" + + +def set_term_width(): + global TERM_WIDTH + + try: + with open(os.devnull, 'w') as NULL: + output = subprocess.check_output(["tput", "cols"], stderr=NULL) + width = int(output) + if width > 60: + TERM_WIDTH = width - 4 + except Exception: + pass + + +def print_help(): + print("usage: software [--debug]") + print(" ...") + print("") + print("Subcomands:") + print("") + print(textwrap.fill(" {0:<15} ".format("deploy start:") + help_upload, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("deploy host:") + help_upload_dir, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("deploy complete:") + help_apply, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print(textwrap.fill(help_patch_args, + width=TERM_WIDTH, initial_indent=' ' * 20, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("release upload:") + help_upload, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print(textwrap.fill(help_patch_args, + width=TERM_WIDTH, initial_indent=' ' * 20, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("release delete:") + help_delete, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print(textwrap.fill(help_patch_args, + width=TERM_WIDTH, initial_indent=' ' * 20, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("release show:") + help_query, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("release list:") + help_show, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("query-dependencies:") + help_query_dependencies, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("is-applied:") + help_is_applied, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("is-available:") + help_is_available, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("report-app-dependencies:") + help_report_app_dependencies, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print("") + print(textwrap.fill(" {0:<15} ".format("query-app-dependencies:") + help_query_app_dependencies, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + print("") + exit(1) + + +def check_rc(req): + rc = 0 + if req.status_code == 200: + data = json.loads(req.text) + if 'error' in data and data["error"] != "": + rc = 1 + else: + rc = 1 + + return rc + + +def print_result_debug(req): + if req.status_code == 200: + data = json.loads(req.text) + if 'pd' in data: + print(json.dumps(data['pd'], + sort_keys=True, + indent=4, + separators=(',', ': '))) + elif 'data' in data: + print(json.dumps(data['data'], + sort_keys=True, + indent=4, + separators=(',', ': '))) + else: + print(json.dumps(data, + sort_keys=True, + indent=4, + separators=(',', ': '))) + elif req.status_code == 500: + print("An internal error has occurred. Please check /var/log/software.log for details") + else: + m = re.search("(Error message:.*)", req.text, re.MULTILINE) + print(m.group(0)) + + +def print_patch_op_result(req): + if req.status_code == 200: + data = json.loads(req.text) + + if 'pd' in data: + pd = data['pd'] + + # Calculate column widths + hdr_id = "Patch ID" + hdr_rr = "RR" + hdr_rel = "Release" + hdr_repo = "Repo State" + hdr_state = "Patch State" + + width_id = len(hdr_id) + width_rr = len(hdr_rr) + width_rel = len(hdr_rel) + width_repo = len(hdr_repo) + width_state = len(hdr_state) + + show_repo = False + + for patch_id in list(pd): + if len(patch_id) > width_id: + width_id = len(patch_id) + if len(pd[patch_id]["sw_version"]) > width_rel: + width_rel = len(pd[patch_id]["sw_version"]) + if len(pd[patch_id]["repostate"]) > width_repo: + width_repo = len(pd[patch_id]["repostate"]) + if len(pd[patch_id]["patchstate"]) > width_state: + width_state = len(pd[patch_id]["patchstate"]) + if pd[patch_id]["patchstate"] == "n/a": + show_repo = True + + if show_repo: + print("{0:^{width_id}} {1:^{width_rr}} {2:^{width_rel}} {3:^{width_repo}} {4:^{width_state}}".format( + hdr_id, hdr_rr, hdr_rel, hdr_repo, hdr_state, + width_id=width_id, width_rr=width_rr, + width_rel=width_rel, width_repo=width_repo, width_state=width_state)) + + print("{0} {1} {2} {3} {4}".format( + '=' * width_id, '=' * width_rr, '=' * width_rel, '=' * width_repo, '=' * width_state)) + + for patch_id in sorted(list(pd)): + if "reboot_required" in pd[patch_id]: + rr = pd[patch_id]["reboot_required"] + else: + rr = "Y" + + print("{0:<{width_id}} {1:^{width_rr}} {2:^{width_rel}} {3:^{width_repo}} {4:^{width_state}}".format( + patch_id, + rr, + pd[patch_id]["sw_version"], + pd[patch_id]["repostate"], + pd[patch_id]["patchstate"], + width_id=width_id, width_rr=width_rr, + width_rel=width_rel, width_repo=width_repo, width_state=width_state)) + else: + print("{0:^{width_id}} {1:^{width_rr}} {2:^{width_rel}} {3:^{width_state}}".format( + hdr_id, hdr_rr, hdr_rel, hdr_state, + width_id=width_id, width_rr=width_rr, width_rel=width_rel, width_state=width_state)) + + print("{0} {1} {2} {3}".format( + '=' * width_id, '=' * width_rr, '=' * width_rel, '=' * width_state)) + + for patch_id in sorted(list(pd)): + if "reboot_required" in pd[patch_id]: + rr = pd[patch_id]["reboot_required"] + else: + rr = "Y" + + print("{0:<{width_id}} {1:^{width_rr}} {2:^{width_rel}} {3:^{width_state}}".format( + patch_id, + rr, + pd[patch_id]["sw_version"], + pd[patch_id]["patchstate"], + width_id=width_id, width_rr=width_rr, width_rel=width_rel, width_state=width_state)) + + print("") + + if 'info' in data and data["info"] != "": + print(data["info"]) + + if 'warning' in data and data["warning"] != "": + print("Warning:") + print(data["warning"]) + + if 'error' in data and data["error"] != "": + print("Error:") + print(data["error"]) + + elif req.status_code == 500: + print("An internal error has occurred. Please check /var/log/software.log for details") + + +def print_patch_show_result(req): + if req.status_code == 200: + data = json.loads(req.text) + + if 'metadata' in data: + pd = data['metadata'] + contents = data['contents'] + for patch_id in sorted(list(pd)): + print("%s:" % patch_id) + + if "sw_version" in pd[patch_id] and pd[patch_id]["sw_version"] != "": + print(textwrap.fill(" {0:<15} ".format("Release:") + pd[patch_id]["sw_version"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + + if "patchstate" in pd[patch_id] and pd[patch_id]["patchstate"] != "": + print(textwrap.fill(" {0:<15} ".format("Patch State:") + pd[patch_id]["patchstate"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + + if pd[patch_id]["patchstate"] == "n/a": + if "repostate" in pd[patch_id] and pd[patch_id]["repostate"] != "": + print(textwrap.fill(" {0:<15} ".format("Repo State:") + pd[patch_id]["repostate"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + + if "status" in pd[patch_id] and pd[patch_id]["status"] != "": + print(textwrap.fill(" {0:<15} ".format("Status:") + pd[patch_id]["status"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + + if "unremovable" in pd[patch_id] and pd[patch_id]["unremovable"] != "": + print(textwrap.fill(" {0:<15} ".format("Unremovable:") + pd[patch_id]["unremovable"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + + if "reboot_required" in pd[patch_id] and pd[patch_id]["reboot_required"] != "": + print(textwrap.fill(" {0:<15} ".format("RR:") + pd[patch_id]["reboot_required"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + + if "apply_active_release_only" in pd[patch_id] and pd[patch_id]["apply_active_release_only"] != "": + print(textwrap.fill(" {0:<15} ".format("Apply Active Release Only:") + pd[patch_id]["apply_active_release_only"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + + if "summary" in pd[patch_id] and pd[patch_id]["summary"] != "": + print(textwrap.fill(" {0:<15} ".format("Summary:") + pd[patch_id]["summary"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + + if "description" in pd[patch_id] and pd[patch_id]["description"] != "": + first_line = True + for line in pd[patch_id]["description"].split('\n'): + if first_line: + print(textwrap.fill(" {0:<15} ".format("Description:") + line, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + first_line = False + else: + print(textwrap.fill(line, + width=TERM_WIDTH, subsequent_indent=' ' * 20, + initial_indent=' ' * 20)) + + if "install_instructions" in pd[patch_id] and pd[patch_id]["install_instructions"] != "": + print(" Install Instructions:") + for line in pd[patch_id]["install_instructions"].split('\n'): + print(textwrap.fill(line, + width=TERM_WIDTH, subsequent_indent=' ' * 20, + initial_indent=' ' * 20)) + + if "warnings" in pd[patch_id] and pd[patch_id]["warnings"] != "": + first_line = True + for line in pd[patch_id]["warnings"].split('\n'): + if first_line: + print(textwrap.fill(" {0:<15} ".format("Warnings:") + line, + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + first_line = False + else: + print(textwrap.fill(line, + width=TERM_WIDTH, subsequent_indent=' ' * 20, + initial_indent=' ' * 20)) + + if "requires" in pd[patch_id] and len(pd[patch_id]["requires"]) > 0: + print(" Requires:") + for req_patch in sorted(pd[patch_id]["requires"]): + print(' ' * 20 + req_patch) + + if "contents" in data and patch_id in data["contents"]: + print(" Contents:\n") + if "number_of_commits" in contents[patch_id] and \ + contents[patch_id]["number_of_commits"] != "": + print(textwrap.fill(" {0:<15} ".format("No. of commits:") + + contents[patch_id]["number_of_commits"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + if "base" in contents[patch_id] and \ + contents[patch_id]["base"]["commit"] != "": + print(textwrap.fill(" {0:<15} ".format("Base commit:") + + contents[patch_id]["base"]["commit"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + if "number_of_commits" in contents[patch_id] and \ + contents[patch_id]["number_of_commits"] != "": + for i in range(int(contents[patch_id]["number_of_commits"])): + print(textwrap.fill(" {0:<15} ".format("Commit%s:" % (i + 1)) + + contents[patch_id]["commit%s" % (i + 1)]["commit"], + width=TERM_WIDTH, subsequent_indent=' ' * 20)) + + print("\n") + + if 'info' in data and data["info"] != "": + print(data["info"]) + + if 'warning' in data and data["warning"] != "": + print("Warning:") + print(data["warning"]) + + if 'error' in data and data["error"] != "": + print("Error:") + print(data["error"]) + + elif req.status_code == 500: + print("An internal error has occurred. Please check /var/log/software.log for details") + + +def patch_upload_req(debug, args): + rc = 0 + + if len(args) == 0: + print_help() + + # Ignore interrupts during this function + signal.signal(signal.SIGINT, signal.SIG_IGN) + + for patchfile in sorted(list(set(args))): + if os.path.isdir(patchfile): + print("Error: %s is a directory. Please use upload-dir" % patchfile) + continue + + if not os.path.isfile(patchfile): + print("Error: File does not exist: %s" % patchfile) + continue + + enc = MultipartEncoder(fields={'file': (patchfile, + open(patchfile, 'rb'), + )}) + url = "http://%s/software/upload" % api_addr + headers = {'Content-Type': enc.content_type} + append_auth_token_if_required(headers) + req = requests.post(url, + data=enc, + headers=headers) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + if check_rc(req) != 0: + rc = 1 + + return rc + + +def patch_apply_req(debug, args): + if len(args) == 0: + print_help() + + # Ignore interrupts during this function + signal.signal(signal.SIGINT, signal.SIG_IGN) + + extra_opts = [] + + if "--skip-semantic" in args: + idx = args.index("--skip-semantic") + + # Get rid of the --skip-semantic + args.pop(idx) + + # Append the extra opts + extra_opts.append("skip-semantic=yes") + + if len(extra_opts) == 0: + extra_opts_str = '' + else: + extra_opts_str = '?%s' % '&'.join(extra_opts) + + patches = "/".join(args) + url = "http://%s/software/apply/%s%s" % (api_addr, patches, extra_opts_str) + + headers = {} + append_auth_token_if_required(headers) + req = requests.post(url, headers=headers) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def patch_remove_req(debug, args): + if len(args) == 0: + print_help() + + # Ignore interrupts during this function + signal.signal(signal.SIGINT, signal.SIG_IGN) + + extra_opts = [] + + # The removeunremovable option is hidden and should not be added to help + # text or customer documentation. It is for emergency use only - under + # supervision of the design team. + if "--removeunremovable" in args: + idx = args.index("--removeunremovable") + + # Get rid of the --removeunremovable + args.pop(idx) + + # Append the extra opts + extra_opts.append('removeunremovable=yes') + + if "--skipappcheck" in args: + idx = args.index("--skipappcheck") + + # Get rid of the --skipappcheck + args.pop(idx) + + # Append the extra opts + extra_opts.append("skipappcheck=yes") + + if "--skip-semantic" in args: + idx = args.index("--skip-semantic") + + # Get rid of the --skip-semantic + args.pop(idx) + + # Append the extra opts + extra_opts.append("skip-semantic=yes") + + if len(extra_opts) == 0: + extra_opts_str = '' + else: + extra_opts_str = '?%s' % '&'.join(extra_opts) + + patches = "/".join(args) + url = "http://%s/software/remove/%s%s" % (api_addr, patches, extra_opts_str) + + headers = {} + append_auth_token_if_required(headers) + req = requests.post(url, headers=headers) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def patch_delete_req(debug, args): + if len(args) == 0: + print_help() + + # Ignore interrupts during this function + signal.signal(signal.SIGINT, signal.SIG_IGN) + + patches = "/".join(args) + + url = "http://%s/software/delete/%s" % (api_addr, patches) + + headers = {} + append_auth_token_if_required(headers) + req = requests.post(url, headers=headers) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def patch_commit_req(debug, args): + if len(args) == 0: + print_help() + + # Ignore interrupts during this function + signal.signal(signal.SIGINT, signal.SIG_IGN) + + dry_run = False + if constants.CLI_OPT_DRY_RUN in args: + dry_run = True + args.remove(constants.CLI_OPT_DRY_RUN) + + all_patches = False + if constants.CLI_OPT_ALL in args: + all_patches = True + args.remove(constants.CLI_OPT_ALL) + + # Default to running release + relopt = RUNNING_SW_VERSION + + release = False + if constants.CLI_OPT_RELEASE in args: + release = True + idx = args.index(constants.CLI_OPT_RELEASE) + # There must be at least one more arg + if len(args) < (idx + 1): + print_help() + + # Get rid of the --release + args.pop(idx) + # Pop off the release arg + relopt = args.pop(idx) + + headers = {} + append_auth_token_if_required(headers) + if release and not all_patches: + # Disallow + print("Use of --release option requires --all") + return 1 + elif all_patches: + # Get a list of all patches + extra_opts = "&release=%s" % relopt + url = "http://%s/software/query?show=all%s" % (api_addr, extra_opts) + + req = requests.get(url, headers=headers) + + patch_list = [] + if req.status_code == 200: + data = json.loads(req.text) + + if 'pd' in data: + patch_list = sorted(list(data['pd'])) + elif req.status_code == 500: + print("Failed to get patch list. Aborting...") + return 1 + + if len(patch_list) == 0: + print("There are no %s patches to commit." % relopt) + return 0 + + print("The following patches will be committed:") + for patch_id in patch_list: + print(" %s" % patch_id) + print() + + patches = "/".join(patch_list) + else: + patches = "/".join(args) + + # First, get a list of dependencies and ask for confirmation + url = "http://%s/software/query_dependencies/%s?recursive=yes" % (api_addr, patches) + + req = requests.get(url, headers=headers) + + if req.status_code == 200: + data = json.loads(req.text) + + if 'patches' in data: + print("The following patches will be committed:") + for patch_id in sorted(data['patches']): + print(" %s" % patch_id) + print() + else: + print("No patches found to commit") + return 1 + + elif req.status_code == 500: + print("An internal error has occurred. Please check /var/log/software.log for details") + return 1 + + # Run dry-run + url = "http://%s/software/commit_dry_run/%s" % (api_addr, patches) + + req = requests.post(url, headers=headers) + print_patch_op_result(req) + + if check_rc(req) != 0: + print("Aborting...") + return 1 + + if dry_run: + return 0 + + print() + commit_warning = "WARNING: Committing a patch is an irreversible operation. " + \ + "Committed patches cannot be removed." + print(textwrap.fill(commit_warning, width=TERM_WIDTH, subsequent_indent=' ' * 9)) + print() + + user_input = input("Would you like to continue? [y/N]: ") + if user_input.lower() != 'y': + print("Aborting...") + return 1 + + url = "http://%s/software/commit/%s" % (api_addr, patches) + req = requests.post(url, headers=headers) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def patch_query_req(debug, args): + state = "all" + extra_opts = "" + + if "--release" in args: + idx = args.index("--release") + # There must be at least one more arg + if len(args) < (idx + 1): + print_help() + + # Get rid of the --release + args.pop(idx) + # Pop off the release arg + relopt = args.pop(idx) + + # Format the query string + extra_opts = "&release=%s" % relopt + + if len(args) > 1: + # Support 1 additional arg at most, currently + print_help() + + if len(args) > 0: + state = args[0] + + url = "http://%s/software/query?show=%s%s" % (api_addr, state, extra_opts) + + headers = {} + append_auth_token_if_required(headers) + req = requests.get(url, headers=headers) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def print_software_deploy_query_result(req): + if req.status_code == 200: + data = json.loads(req.text) + if 'data' not in data: + print("Invalid data returned:") + print_result_debug(req) + return + + agents = data['data'] + + # Calculate column widths + hdr_hn = "Hostname" + hdr_ip = "IP Address" + hdr_pc = "Patch Current" + hdr_rr = "Reboot Required" + hdr_rel = "Release" + hdr_state = "State" + + width_hn = len(hdr_hn) + width_ip = len(hdr_ip) + width_pc = len(hdr_pc) + width_rr = len(hdr_rr) + width_rel = len(hdr_rel) + width_state = len(hdr_state) + + for agent in sorted(agents, key=lambda a: a["hostname"]): + if len(agent["hostname"]) > width_hn: + width_hn = len(agent["hostname"]) + if len(agent["ip"]) > width_ip: + width_ip = len(agent["ip"]) + if len(agent["sw_version"]) > width_rel: + width_rel = len(agent["sw_version"]) + if len(agent["state"]) > width_state: + width_state = len(agent["state"]) + + print("{0:^{width_hn}} {1:^{width_ip}} {2:^{width_pc}} {3:^{width_rr}} {4:^{width_rel}} {5:^{width_state}}".format( + hdr_hn, hdr_ip, hdr_pc, hdr_rr, hdr_rel, hdr_state, + width_hn=width_hn, width_ip=width_ip, width_pc=width_pc, width_rr=width_rr, width_rel=width_rel, width_state=width_state)) + + print("{0} {1} {2} {3} {4} {5}".format( + '=' * width_hn, '=' * width_ip, '=' * width_pc, '=' * width_rr, '=' * width_rel, '=' * width_state)) + + for agent in sorted(agents, key=lambda a: a["hostname"]): + patch_current_field = "Yes" if agent["patch_current"] else "No" + if agent.get("interim_state") is True: + patch_current_field = "Pending" + + if agent["patch_failed"]: + patch_current_field = "Failed" + + print("{0:<{width_hn}} {1:<{width_ip}} {2:^{width_pc}} {3:^{width_rr}} {4:^{width_rel}} {5:^{width_state}}".format( + agent["hostname"], + agent["ip"], + patch_current_field, + "Yes" if agent["requires_reboot"] else "No", + agent["sw_version"], + agent["state"], + width_hn=width_hn, width_ip=width_ip, width_pc=width_pc, width_rr=width_rr, width_rel=width_rel, width_state=width_state)) + + elif req.status_code == 500: + print("An internal error has occurred. Please check /var/log/software.log for details") + + +def patch_query_hosts_req(debug, args): + if len(args) > 0: + # Support 0 arg at most, currently + print_help() + + url = "http://%s/software/query_hosts" % api_addr + + req = requests.get(url) + + if debug: + print_result_debug(req) + else: + print_software_deploy_query_result(req) + + return check_rc(req) + + +def patch_show_req(debug, args): + if len(args) == 0: + print_help() + + patches = "/".join(args) + + url = "http://%s/software/show/%s" % (api_addr, patches) + + headers = {} + append_auth_token_if_required(headers) + req = requests.post(url, headers=headers) + + if debug: + print_result_debug(req) + else: + print_patch_show_result(req) + + return check_rc(req) + + +def what_requires(debug, args): + if len(args) == 0: + print_help() + + patches = "/".join(args) + + url = "http://%s/software/what_requires/%s" % (api_addr, patches) + + headers = {} + append_auth_token_if_required(headers) + req = requests.get(url, headers=headers) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def query_dependencies(debug, args): + if len(args) == 0: + print_help() + + extra_opts = "" + if constants.CLI_OPT_RECURSIVE in args: + args.remove(constants.CLI_OPT_RECURSIVE) + extra_opts = "?recursive=yes" + + patches = "/".join(args) + + url = "http://%s/software/query_dependencies/%s%s" % (api_addr, patches, extra_opts) + + headers = {} + append_auth_token_if_required(headers) + req = requests.get(url, headers=headers) + + if debug: + print_result_debug(req) + else: + if req.status_code == 200: + data = json.loads(req.text) + + if 'patches' in data: + for patch_id in sorted(data['patches']): + print(patch_id) + if 'error' in data and data["error"] != "": + print("Error: %s" % data.get("error")) + + elif req.status_code == 500: + print("An internal error has occurred. Please check /var/log/software.log for details") + + return check_rc(req) + + +def wait_for_install_complete(agent_ip): + url = "http://%s/software/query_hosts" % api_addr + rc = 0 + + max_retries = 4 + retriable_count = 0 + + while True: + # Sleep on the first pass as well, to allow time for the + # agent to respond + time.sleep(5) + + try: + req = requests.get(url) + except requests.exceptions.ConnectionError: + # The local software-controller may have restarted. + retriable_count += 1 + if retriable_count <= max_retries: + continue + else: + print("Lost communications with the software controller") + rc = 1 + break + + if req.status_code == 200: + data = json.loads(req.text) + if 'data' not in data: + print("Invalid query-hosts data returned:") + print_result_debug(req) + rc = 1 + break + + state = None + agents = data['data'] + interim_state = None + + for agent in agents: + if agent['hostname'] == agent_ip \ + or agent['ip'] == agent_ip: + state = agent.get('state') + interim_state = agent.get('interim_state') + + if state is None: + # If the software daemons have restarted, there's a + # window after the software-controller restart that the + # hosts table will be empty. + retriable_count += 1 + if retriable_count <= max_retries: + continue + else: + print("%s agent has timed out." % agent_ip) + rc = 1 + break + + if state == constants.PATCH_AGENT_STATE_INSTALLING or \ + interim_state is True: + # Still installing + sys.stdout.write(".") + sys.stdout.flush() + elif state == constants.PATCH_AGENT_STATE_INSTALL_REJECTED: + print("\nInstallation rejected. Node must be locked") + rc = 1 + break + elif state == constants.PATCH_AGENT_STATE_INSTALL_FAILED: + print("\nInstallation failed. Please check logs for details.") + rc = 1 + break + elif state == constants.PATCH_AGENT_STATE_IDLE: + print("\nInstallation was successful.") + rc = 0 + break + else: + print("\nPatch agent is reporting unknown state: %s" % state) + rc = 1 + break + + elif req.status_code == 500: + print("An internal error has occurred. Please check /var/log/software.log for details") + rc = 1 + break + else: + m = re.search("(Error message:.*)", req.text, re.MULTILINE) + print(m.group(0)) + rc = 1 + break + + return rc + + +def host_install(debug, args): # pylint: disable=unused-argument + force = False + rc = 0 + + if "--force" in args: + force = True + args.remove("--force") + + if len(args) != 1: + print_help() + + agent_ip = args[0] + + # Issue host_install_async request and poll for results + url = "http://%s/software/host_install_async/%s" % (api_addr, agent_ip) + + if force: + url += "/force" + + req = requests.post(url) + + if req.status_code == 200: + data = json.loads(req.text) + if 'error' in data and data["error"] != "": + print("Error:") + print(data["error"]) + rc = 1 + else: + rc = wait_for_install_complete(agent_ip) + elif req.status_code == 500: + print("An internal error has occurred. " + "Please check /var/log/software.log for details") + rc = 1 + else: + m = re.search("(Error message:.*)", req.text, re.MULTILINE) + print(m.group(0)) + rc = 1 + + return rc + + +def host_install_async(debug, args): + force = False + + if "--force" in args: + force = True + args.remove("--force") + + if len(args) != 1: + print_help() + + agent_ip = args[0] + + url = "http://%s/software/host_install_async/%s" % (api_addr, agent_ip) + + if force: + url += "/force" + + req = requests.post(url) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def drop_host(debug, args): + if len(args) != 1: + print_help() + + host_ip = args[0] + + url = "http://%s/software/drop_host/%s" % (api_addr, host_ip) + + req = requests.post(url) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def patch_upload_dir_req(debug, args): + if len(args) == 0: + print_help() + + # Ignore interrupts during this function + signal.signal(signal.SIGINT, signal.SIG_IGN) + + dirlist = {} + i = 0 + for d in sorted(list(set(args))): + dirlist["dir%d" % i] = os.path.abspath(d) + i += 1 + + url = "http://%s/software/upload_dir" % api_addr + + headers = {} + append_auth_token_if_required(headers) + req = requests.post(url, params=dirlist, headers=headers) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def software_deploy_host_req(debug, args): # pylint: disable=unused-argument + force = False + rc = 0 + + if "--force" in args: + force = True + args.remove("--force") + + if len(args) != 1: + print_help() + + agent_ip = args[0] + + # Issue host_install_async request and poll for results + url = "http://%s/software/host_install_async/%s" % (api_addr, agent_ip) + + if force: + url += "/force" + + req = requests.post(url) + + if req.status_code == 200: + data = json.loads(req.text) + if 'error' in data and data["error"] != "": + print("Error:") + print(data["error"]) + rc = 1 + else: + rc = wait_for_install_complete(agent_ip) + elif req.status_code == 500: + print("An internal error has occurred. " + "Please check /var/log/software.log for details") + rc = 1 + else: + m = re.search("(Error message:.*)", req.text, re.MULTILINE) + print(m.group(0)) + rc = 1 + + return rc + + +def patch_init_release(debug, args): + if len(args) != 1: + print_help() + + # Ignore interrupts during this function + signal.signal(signal.SIGINT, signal.SIG_IGN) + + release = args[0] + + url = "http://%s/software/init_release/%s" % (api_addr, release) + + req = requests.post(url) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def patch_del_release(debug, args): + if len(args) != 1: + print_help() + + # Ignore interrupts during this function + signal.signal(signal.SIGINT, signal.SIG_IGN) + + release = args[0] + + url = "http://%s/software/del_release/%s" % (api_addr, release) + + req = requests.post(url) + + if debug: + print_result_debug(req) + else: + print_patch_op_result(req) + + return check_rc(req) + + +def patch_is_applied_req(args): + if len(args) == 0: + print_help() + + patches = "/".join(args) + url = "http://%s/software/is_applied/%s" % (api_addr, patches) + + headers = {} + append_auth_token_if_required(headers) + req = requests.post(url, headers=headers) + + rc = 1 + + if req.status_code == 200: + result = json.loads(req.text) + print(result) + if result is True: + rc = 0 + elif req.status_code == 500: + print("An internal error has occurred. Please check /var/log/software.log for details") + + return rc + + +def patch_is_available_req(args): + if len(args) == 0: + print_help() + + patches = "/".join(args) + url = "http://%s/software/is_available/%s" % (api_addr, patches) + + headers = {} + append_auth_token_if_required(headers) + req = requests.post(url, headers=headers) + + rc = 1 + + if req.status_code == 200: + result = json.loads(req.text) + print(result) + if result is True: + rc = 0 + elif req.status_code == 500: + print("An internal error has occurred. Please check /var/log/software.log for details") + + return rc + + +def patch_report_app_dependencies_req(debug, args): # pylint: disable=unused-argument + if len(args) < 2: + print_help() + + extra_opts = [] + + if "--app" in args: + idx = args.index("--app") + + # Get rid of the --app and get the app name + args.pop(idx) + app = args.pop(idx) + + # Append the extra opts + extra_opts.append("app=%s" % app) + else: + print("Application name must be specified with --app argument.") + return 1 + + extra_opts_str = '?%s' % '&'.join(extra_opts) + + patches = "/".join(args) + url = "http://%s/software/report_app_dependencies/%s%s" \ + % (api_addr, patches, extra_opts_str) + + headers = {} + append_auth_token_if_required(headers) + req = requests.post(url, headers=headers) + + if req.status_code == 200: + return 0 + else: + print("An internal error has occurred. " + "Please check /var/log/software.log for details.") + return 1 + + +def patch_query_app_dependencies_req(): + url = "http://%s/software/query_app_dependencies" % api_addr + + headers = {} + append_auth_token_if_required(headers) + req = requests.post(url, headers=headers) + + if req.status_code == 200: + data = json.loads(req.text) + if len(data) == 0: + print("There are no application dependencies.") + else: + hdr_app = "Application" + hdr_list = "Required Patches" + width_app = len(hdr_app) + width_list = len(hdr_list) + + for app, patch_list in data.items(): + width_app = max(width_app, len(app)) + width_list = max(width_list, len(', '.join(patch_list))) + + print("{0:<{width_app}} {1:<{width_list}}".format( + hdr_app, hdr_list, + width_app=width_app, width_list=width_list)) + + print("{0} {1}".format( + '=' * width_app, '=' * width_list)) + + for app, patch_list in sorted(data.items()): + print("{0:<{width_app}} {1:<{width_list}}".format( + app, ', '.join(patch_list), + width_app=width_app, width_list=width_list)) + + return 0 + else: + print("An internal error has occurred. " + "Please check /var/log/software.log for details.") + return 1 + + +def check_env(env, var): + if env not in os.environ: + print("You must provide a %s via env[%s]" % (var, env)) + exit(-1) + + +def get_auth_token_and_endpoint(region_name): + from keystoneauth1 import exceptions + from keystoneauth1 import identity + from keystoneauth1 import session + + user_env_map = {'OS_USERNAME': 'username', + 'OS_PASSWORD': 'password', + 'OS_PROJECT_NAME': 'project_name', + 'OS_AUTH_URL': 'auth_url', + 'OS_USER_DOMAIN_NAME': 'user_domain_name', + 'OS_PROJECT_DOMAIN_NAME': 'project_domain_name'} + + for k, v in user_env_map.items(): + check_env(k, v) + + user = dict() + for k, v in user_env_map.items(): + user[v] = os.environ.get(k) + + auth = identity.V3Password(**user) + sess = session.Session(auth=auth) + try: + token = auth.get_token(sess) + endpoint = auth.get_endpoint(sess, service_type='software', + interface='internal', + region_name=region_name) + except (exceptions.http.Unauthorized, exceptions.EndpointNotFound) as e: + print(str(e)) + exit(-1) + + return token, endpoint + + +def append_auth_token_if_required(headers): + global auth_token + if auth_token is not None: + headers['X-Auth-Token'] = auth_token + + +def format_url_address(address): + import netaddr + try: + ip_addr = netaddr.IPAddress(address) + if ip_addr.version == IPV6_FAMILY: + return "[%s]" % address + else: + return address + except netaddr.AddrFormatError: + return address + + +def check_for_os_region_name(): + region_option = "--os-region-name" + if region_option not in sys.argv: + return False + + for c, value in enumerate(sys.argv, 1): + if value == region_option: + if c == len(sys.argv): + print("Please specify a region name") + print_help() + + region = sys.argv[c] + global VIRTUAL_REGION + if region != VIRTUAL_REGION: + print("Unsupported region name: %s" % region) + exit(1) + + # check it is running on the active controller + # not able to use sm-query due to it requires sudo + try: + subprocess.check_output("pgrep -f dcorch-api-proxy", shell=True) + except subprocess.CalledProcessError: + print("Command must be run from the active controller.") + exit(1) + + # get a token and fetch the internal endpoint in SystemController + global auth_token + auth_token, endpoint = get_auth_token_and_endpoint(region) + if endpoint is not None: + global api_addr + try: + # python 2 + from urlparse import urlparse + except ImportError: + # python 3 + from urllib.parse import urlparse + url = urlparse(endpoint) + address = format_url_address(url.hostname) + api_addr = '{}:{}'.format(address, url.port) + + sys.argv.remove("--os-region-name") + sys.argv.remove(region) + return True + + +def main(): + set_term_width() + + if len(sys.argv) <= 1: + print_help() + + debug = False + if "--debug" in sys.argv: + debug = True + sys.argv.remove("--debug") + + dc_request = check_for_os_region_name() + + rc = 0 + + action = sys.argv[1] + + # Reject the commands that are not supported in the virtual region + if (dc_request and action in ["deploy host", "deploy start", + "deploy list", "deploy query", + "deploy activate", "deploy complete"]): + global VIRTUAL_REGION + print("\n%s command is not allowed in %s region" % (action, + VIRTUAL_REGION)) + exit(1) + + if auth_token is None and os.geteuid() != 0: + # Restrict non-root/sudo users to these commands + if action == "release list": + print_help() + elif action == "release show": + print_help() + elif action == "deploy list": + print_help() + elif action == "deploy query": + print_help() + elif action == "--help" or action == "-h": + print_help() + else: + print("Error: Command must be run as sudo or root", file=sys.stderr) + rc = 1 + else: + if action == "release upload": + print_help() + if action == "release upload-dir": + print_help() + elif action == "release delete": + print_help() + elif action == "deploy create": + print_help() + elif action == "deploy delete": + print_help() + elif action == "deploy precheck": + print_help() + elif action == "deploy start": + print_help() + elif action == "deploy host": + print_help() + elif action == "deploy activate": + print_help() + elif action == "deploy complete": + print_help() + elif action == "deploy abort": + print_help() + elif action == "deploy host-rollback": + print_help() + elif action == "is-applied": + rc = patch_is_applied_req(sys.argv[2:]) + elif action == "is-available": + rc = patch_is_available_req(sys.argv[2:]) + elif action == "report-app-dependencies": + rc = patch_report_app_dependencies_req(debug, sys.argv[2:]) + elif action == "query-app-dependencies": + rc = patch_query_app_dependencies_req() + else: + print_help() + + exit(rc) diff --git a/software/software/software_config.py b/software/software/software_config.py new file mode 100644 index 00000000..2d2715d7 --- /dev/null +++ b/software/software/software_config.py @@ -0,0 +1,124 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" +import configparser +import io +import logging +import os +import socket + +import tsconfig.tsconfig as tsc + +import software.utils as utils +import software.constants as constants + +controller_mcast_group = None +agent_mcast_group = None +controller_port = 0 +agent_port = 0 +api_port = 0 +mgmt_if = None +nodetype = None +platform_conf_mtime = 0 +software_conf_mtime = 0 +software_conf = '/etc/software/software.conf' + + +def read_config(): + global software_conf_mtime + global software_conf + + if software_conf_mtime == os.stat(software_conf).st_mtime: + # The file has not changed since it was last read + return + + defaults = { + 'controller_mcast_group': "239.1.1.3", + 'agent_mcast_group': "239.1.1.4", + 'api_port': "5493", + 'controller_port': "5494", + 'agent_port': "5495", + } + + global controller_mcast_group + global agent_mcast_group + global api_port + global controller_port + global agent_port + + config = configparser.ConfigParser(defaults) + + config.read(software_conf) + software_conf_mtime = os.stat(software_conf).st_mtime + + controller_mcast_group = config.get('runtime', + 'controller_multicast') + agent_mcast_group = config.get('runtime', 'agent_multicast') + + api_port = config.getint('runtime', 'api_port') + controller_port = config.getint('runtime', 'controller_port') + agent_port = config.getint('runtime', 'agent_port') + + # The platform.conf file has no section headers, which causes problems + # for ConfigParser. So we'll fake it out. + ini_str = '[platform_conf]\n' + open(tsc.PLATFORM_CONF_FILE, 'r').read() + ini_fp = io.StringIO(ini_str) + config.read_file(ini_fp) + + try: + value = str(config.get('platform_conf', 'nodetype')) + + global nodetype + nodetype = value + except configparser.Error: + logging.exception("Failed to read nodetype from config") + + +def get_mgmt_ip(): + # Check if initial config is complete + if not os.path.exists('/etc/platform/.initial_config_complete'): + return None + mgmt_hostname = socket.gethostname() + return utils.gethostbyname(mgmt_hostname) + + +# Because the software daemons are launched before manifests are +# applied, the content of some settings in platform.conf can change, +# such as the management interface. As such, we can't just directly +# use tsc.management_interface +# +def get_mgmt_iface(): + # Check if initial config is complete + if not os.path.exists(constants.INITIAL_CONFIG_COMPLETE_FLAG): + return None + + global mgmt_if + global platform_conf_mtime + + if mgmt_if is not None and \ + platform_conf_mtime == os.stat(tsc.PLATFORM_CONF_FILE).st_mtime: + # The platform.conf file hasn't been modified since we read it, + # so return the cached value. + return mgmt_if + + config = configparser.ConfigParser() + + # The platform.conf file has no section headers, which causes problems + # for ConfigParser. So we'll fake it out. + ini_str = '[platform_conf]\n' + open(tsc.PLATFORM_CONF_FILE, 'r').read() + ini_fp = io.StringIO(ini_str) + config.read_file(ini_fp) + + try: + value = str(config.get('platform_conf', 'management_interface')) + + mgmt_if = value + + platform_conf_mtime = os.stat(tsc.PLATFORM_CONF_FILE).st_mtime + except configparser.Error: + logging.exception("Failed to read management_interface from config") + return None + return mgmt_if diff --git a/software/software/software_controller.py b/software/software/software_controller.py new file mode 100644 index 00000000..8dff14d9 --- /dev/null +++ b/software/software/software_controller.py @@ -0,0 +1,2711 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" +import sys + +# prevent software_controller from importing osprofiler +sys.modules['osprofiler'] = None + +import configparser +import gc +import json +import os +import select +import sh +import shutil +import socket +import subprocess +import tarfile +import tempfile +import threading +import time +from wsgiref import simple_server + +from oslo_config import cfg as oslo_cfg + +from software import ostree_utils +from software.api import app +from software.authapi import app as auth_app +from software.base import PatchService +from software.exceptions import MetadataFail +from software.exceptions import OSTreeCommandFail +from software.exceptions import OSTreeTarFail +from software.exceptions import PatchError +from software.exceptions import PatchFail +from software.exceptions import PatchInvalidRequest +from software.exceptions import PatchValidationFailure +from software.exceptions import PatchMismatchFailure +from software.exceptions import SemanticFail +from software.software_functions import configure_logging +from software.software_functions import BasePackageData +from software.software_functions import avail_dir +from software.software_functions import applied_dir +from software.software_functions import committed_dir +from software.software_functions import PatchFile +from software.software_functions import package_dir +from software.software_functions import repo_dir +from software.software_functions import root_scripts_dir +from software.software_functions import semantics_dir +from software.software_functions import SW_VERSION +from software.software_functions import root_package_dir +from software.software_functions import LOG +from software.software_functions import audit_log_info +from software.software_functions import patch_dir +from software.software_functions import repo_root_dir +from software.software_functions import PatchData + +import software.software_config as cfg +import software.utils as utils + +import software.messages as messages +import software.constants as constants + +from tsconfig.tsconfig import INITIAL_CONFIG_COMPLETE_FLAG + +CONF = oslo_cfg.CONF + +pidfile_path = "/var/run/patch_controller.pid" + +pc = None +state_file = "%s/.controller.state" % constants.PATCH_STORAGE_DIR +app_dependency_basename = "app_dependencies.json" +app_dependency_filename = "%s/%s" % (constants.PATCH_STORAGE_DIR, app_dependency_basename) + +insvc_patch_restart_controller = "/run/software/.restart.software-controller" + +stale_hosts = [] +pending_queries = [] + +thread_death = None +keep_running = True + +# Limit socket blocking to 5 seconds to allow for thread to shutdown +api_socket_timeout = 5.0 + + +class ControllerNeighbour(object): + def __init__(self): + self.last_ack = 0 + self.synced = False + + def rx_ack(self): + self.last_ack = time.time() + + def get_age(self): + return int(time.time() - self.last_ack) + + def rx_synced(self): + self.synced = True + + def clear_synced(self): + self.synced = False + + def get_synced(self): + return self.synced + + +class AgentNeighbour(object): + def __init__(self, ip): + self.ip = ip + self.last_ack = 0 + self.last_query_id = 0 + self.out_of_date = False + self.hostname = "n/a" + self.requires_reboot = False + self.patch_failed = False + self.stale = False + self.pending_query = False + self.latest_sysroot_commit = None + self.nodetype = None + self.sw_version = "unknown" + self.subfunctions = [] + self.state = None + + def rx_ack(self, + hostname, + out_of_date, + requires_reboot, + query_id, + patch_failed, + sw_version, + state): + self.last_ack = time.time() + self.hostname = hostname + self.patch_failed = patch_failed + self.sw_version = sw_version + self.state = state + + if out_of_date != self.out_of_date or requires_reboot != self.requires_reboot: + self.out_of_date = out_of_date + self.requires_reboot = requires_reboot + LOG.info("Agent %s (%s) reporting out_of_date=%s, requires_reboot=%s", + self.hostname, + self.ip, + self.out_of_date, + self.requires_reboot) + + if self.last_query_id != query_id: + self.last_query_id = query_id + self.stale = True + if self.ip not in stale_hosts and self.ip not in pending_queries: + stale_hosts.append(self.ip) + + def get_age(self): + return int(time.time() - self.last_ack) + + def handle_query_detailed_resp(self, + latest_sysroot_commit, + nodetype, + sw_version, + subfunctions, + state): + self.latest_sysroot_commit = latest_sysroot_commit + self.nodetype = nodetype + self.stale = False + self.pending_query = False + self.sw_version = sw_version + self.subfunctions = subfunctions + self.state = state + + if self.ip in pending_queries: + pending_queries.remove(self.ip) + + if self.ip in stale_hosts: + stale_hosts.remove(self.ip) + + def get_dict(self): + d = {"ip": self.ip, + "hostname": self.hostname, + "patch_current": not self.out_of_date, + "secs_since_ack": self.get_age(), + "patch_failed": self.patch_failed, + "stale_details": self.stale, + "latest_sysroot_commit": self.latest_sysroot_commit, + "nodetype": self.nodetype, + "subfunctions": self.subfunctions, + "sw_version": self.sw_version, + "state": self.state} + + global pc + if self.out_of_date and not pc.allow_insvc_patching: + d["requires_reboot"] = True + else: + d["requires_reboot"] = self.requires_reboot + + # Included for future enhancement, to allow per-node determination + # of in-service patching + d["allow_insvc_patching"] = pc.allow_insvc_patching + + return d + + +class PatchMessageHello(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_HELLO) + self.patch_op_counter = 0 + + def decode(self, data): + messages.PatchMessage.decode(self, data) + if 'patch_op_counter' in data: + self.patch_op_counter = data['patch_op_counter'] + + def encode(self): + global pc + messages.PatchMessage.encode(self) + self.message['patch_op_counter'] = pc.patch_op_counter + + def handle(self, sock, addr): + global pc + host = addr[0] + if host == cfg.get_mgmt_ip(): + # Ignore messages from self + return + + # Send response + if self.patch_op_counter > 0: + pc.handle_nbr_patch_op_counter(host, self.patch_op_counter) + + resp = PatchMessageHelloAck() + resp.send(sock) + + def send(self, sock): + global pc + self.encode() + message = json.dumps(self.message) + sock.sendto(str.encode(message), (pc.controller_address, cfg.controller_port)) + + +class PatchMessageHelloAck(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_HELLO_ACK) + + def encode(self): + # Nothing to add, so just call the super class + messages.PatchMessage.encode(self) + + def handle(self, sock, addr): + global pc + + pc.controller_neighbours_lock.acquire() + if not addr[0] in pc.controller_neighbours: + pc.controller_neighbours[addr[0]] = ControllerNeighbour() + + pc.controller_neighbours[addr[0]].rx_ack() + pc.controller_neighbours_lock.release() + + def send(self, sock): + global pc + self.encode() + message = json.dumps(self.message) + sock.sendto(str.encode(message), (pc.controller_address, cfg.controller_port)) + + +class PatchMessageSyncReq(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_SYNC_REQ) + + def encode(self): + # Nothing to add to the SYNC_REQ, so just call the super class + messages.PatchMessage.encode(self) + + def handle(self, sock, addr): + global pc + host = addr[0] + if host == cfg.get_mgmt_ip(): + # Ignore messages from self + return + + # We may need to do this in a separate thread, so that we continue to process hellos + LOG.info("Handling sync req") + + pc.sync_from_nbr(host) + + resp = PatchMessageSyncComplete() + resp.send(sock) + + def send(self, sock): + global pc + LOG.info("sending sync req") + self.encode() + message = json.dumps(self.message) + sock.sendto(str.encode(message), (pc.controller_address, cfg.controller_port)) + + +class PatchMessageSyncComplete(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_SYNC_COMPLETE) + + def encode(self): + # Nothing to add to the SYNC_COMPLETE, so just call the super class + messages.PatchMessage.encode(self) + + def handle(self, sock, addr): + global pc + LOG.info("Handling sync complete") + + pc.controller_neighbours_lock.acquire() + if not addr[0] in pc.controller_neighbours: + pc.controller_neighbours[addr[0]] = ControllerNeighbour() + + pc.controller_neighbours[addr[0]].rx_synced() + pc.controller_neighbours_lock.release() + + def send(self, sock): + global pc + LOG.info("sending sync complete") + self.encode() + message = json.dumps(self.message) + sock.sendto(str.encode(message), (pc.controller_address, cfg.controller_port)) + + +class PatchMessageHelloAgent(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_HELLO_AGENT) + + def encode(self): + global pc + messages.PatchMessage.encode(self) + self.message['patch_op_counter'] = pc.patch_op_counter + + def handle(self, sock, addr): + LOG.error("Should not get here") + + def send(self, sock): + global pc + self.encode() + message = json.dumps(self.message) + local_hostname = utils.ip_to_versioned_localhost(cfg.agent_mcast_group) + sock.sendto(str.encode(message), (pc.agent_address, cfg.agent_port)) + sock.sendto(str.encode(message), (local_hostname, cfg.agent_port)) + + +class PatchMessageSendLatestFeedCommit(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_SEND_LATEST_FEED_COMMIT) + + def encode(self): + global pc + messages.PatchMessage.encode(self) + self.message['latest_feed_commit'] = pc.latest_feed_commit + + def handle(self, sock, addr): + LOG.error("Should not get here") + + def send(self, sock): + global pc + self.encode() + message = json.dumps(self.message) + local_hostname = utils.ip_to_versioned_localhost(cfg.agent_mcast_group) + sock.sendto(str.encode(message), (pc.agent_address, cfg.agent_port)) + sock.sendto(str.encode(message), (local_hostname, cfg.agent_port)) + + +class PatchMessageHelloAgentAck(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_HELLO_AGENT_ACK) + self.query_id = 0 + self.agent_out_of_date = False + self.agent_hostname = "n/a" + self.agent_requires_reboot = False + self.agent_patch_failed = False + self.agent_sw_version = "unknown" + self.agent_state = "unknown" + + def decode(self, data): + messages.PatchMessage.decode(self, data) + if 'query_id' in data: + self.query_id = data['query_id'] + if 'out_of_date' in data: + self.agent_out_of_date = data['out_of_date'] + if 'hostname' in data: + self.agent_hostname = data['hostname'] + if 'requires_reboot' in data: + self.agent_requires_reboot = data['requires_reboot'] + if 'patch_failed' in data: + self.agent_patch_failed = data['patch_failed'] + if 'sw_version' in data: + self.agent_sw_version = data['sw_version'] + if 'state' in data: + self.agent_state = data['state'] + + def encode(self): + # Nothing to add, so just call the super class + messages.PatchMessage.encode(self) + + def handle(self, sock, addr): + global pc + + pc.hosts_lock.acquire() + if not addr[0] in pc.hosts: + pc.hosts[addr[0]] = AgentNeighbour(addr[0]) + + pc.hosts[addr[0]].rx_ack(self.agent_hostname, + self.agent_out_of_date, + self.agent_requires_reboot, + self.query_id, + self.agent_patch_failed, + self.agent_sw_version, + self.agent_state) + pc.hosts_lock.release() + + def send(self, sock): # pylint: disable=unused-argument + LOG.error("Should not get here") + + +class PatchMessageQueryDetailed(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_QUERY_DETAILED) + + def encode(self): + # Nothing to add to the message, so just call the super class + messages.PatchMessage.encode(self) + + def handle(self, sock, addr): + LOG.error("Should not get here") + + def send(self, sock): + self.encode() + message = json.dumps(self.message) + sock.sendall(str.encode(message)) + + +class PatchMessageQueryDetailedResp(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_QUERY_DETAILED_RESP) + self.agent_sw_version = "unknown" + self.latest_sysroot_commit = "unknown" + self.subfunctions = [] + self.nodetype = "unknown" + self.agent_sw_version = "unknown" + self.agent_state = "unknown" + + def decode(self, data): + messages.PatchMessage.decode(self, data) + if 'latest_sysroot_commit' in data: + self.latest_sysroot_commit = data['latest_sysroot_commit'] + if 'nodetype' in data: + self.nodetype = data['nodetype'] + if 'sw_version' in data: + self.agent_sw_version = data['sw_version'] + if 'subfunctions' in data: + self.subfunctions = data['subfunctions'] + if 'state' in data: + self.agent_state = data['state'] + + def encode(self): + LOG.error("Should not get here") + + def handle(self, sock, addr): + global pc + + ip = addr[0] + pc.hosts_lock.acquire() + if ip in pc.hosts: + pc.hosts[ip].handle_query_detailed_resp(self.latest_sysroot_commit, + self.nodetype, + self.agent_sw_version, + self.subfunctions, + self.agent_state) + for patch_id in list(pc.interim_state): + if ip in pc.interim_state[patch_id]: + pc.interim_state[patch_id].remove(ip) + if len(pc.interim_state[patch_id]) == 0: + del pc.interim_state[patch_id] + pc.hosts_lock.release() + pc.check_patch_states() + else: + pc.hosts_lock.release() + + def send(self, sock): # pylint: disable=unused-argument + LOG.error("Should not get here") + + +class PatchMessageAgentInstallReq(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_AGENT_INSTALL_REQ) + self.ip = None + self.force = False + + def encode(self): + global pc + messages.PatchMessage.encode(self) + self.message['force'] = self.force + + def handle(self, sock, addr): + LOG.error("Should not get here") + + def send(self, sock): + LOG.info("sending install request to node: %s", self.ip) + self.encode() + message = json.dumps(self.message) + sock.sendto(str.encode(message), (self.ip, cfg.agent_port)) + + +class PatchMessageAgentInstallResp(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_AGENT_INSTALL_RESP) + self.status = False + self.reject_reason = None + + def decode(self, data): + messages.PatchMessage.decode(self, data) + if 'status' in data: + self.status = data['status'] + if 'reject_reason' in data: + self.reject_reason = data['reject_reason'] + + def encode(self): + # Nothing to add, so just call the super class + messages.PatchMessage.encode(self) + + def handle(self, sock, addr): + LOG.info("Handling install resp from %s", addr[0]) + global pc + # LOG.info("Handling hello ack") + + pc.hosts_lock.acquire() + if not addr[0] in pc.hosts: + pc.hosts[addr[0]] = AgentNeighbour(addr[0]) + + pc.hosts[addr[0]].install_status = self.status + pc.hosts[addr[0]].install_pending = False + pc.hosts[addr[0]].install_reject_reason = self.reject_reason + pc.hosts_lock.release() + + def send(self, sock): # pylint: disable=unused-argument + LOG.error("Should not get here") + + +class PatchMessageDropHostReq(messages.PatchMessage): + def __init__(self): + messages.PatchMessage.__init__(self, messages.PATCHMSG_DROP_HOST_REQ) + self.ip = None + + def encode(self): + messages.PatchMessage.encode(self) + self.message['ip'] = self.ip + + def decode(self, data): + messages.PatchMessage.decode(self, data) + if 'ip' in data: + self.ip = data['ip'] + + def handle(self, sock, addr): + global pc + host = addr[0] + if host == cfg.get_mgmt_ip(): + # Ignore messages from self + return + + if self.ip is None: + LOG.error("Received PATCHMSG_DROP_HOST_REQ with no ip: %s", json.dumps(self.data)) + return + + pc.drop_host(self.ip, sync_nbr=False) + return + + def send(self, sock): + global pc + self.encode() + message = json.dumps(self.message) + sock.sendto(str.encode(message), (pc.controller_address, cfg.controller_port)) + + +class PatchController(PatchService): + def __init__(self): + PatchService.__init__(self) + + # Locks + self.socket_lock = threading.RLock() + self.controller_neighbours_lock = threading.RLock() + self.hosts_lock = threading.RLock() + self.patch_data_lock = threading.RLock() + + self.hosts = {} + self.controller_neighbours = {} + + # interim_state is used to track hosts that have not responded + # with fresh queries since a patch was applied or removed, on + # a per-patch basis. This allows the patch controller to move + # patches immediately into a "Partial" state until all nodes + # have responded. + # + self.interim_state = {} + + self.sock_out = None + self.sock_in = None + self.controller_address = None + self.agent_address = None + self.patch_op_counter = 1 + self.patch_data = PatchData() + self.patch_data.load_all() + try: + self.latest_feed_commit = ostree_utils.get_feed_latest_commit(SW_VERSION) + except OSTreeCommandFail: + LOG.exception("Failure to fetch the feed ostree latest log while " + "initializing Patch Controller") + self.latest_feed_commit = None + + self.check_patch_states() + self.base_pkgdata = BasePackageData() + + self.allow_insvc_patching = True + + if os.path.exists(app_dependency_filename): + try: + with open(app_dependency_filename, 'r') as f: + self.app_dependencies = json.loads(f.read()) + except Exception: + LOG.exception("Failed to read app dependencies: %s", app_dependency_filename) + else: + self.app_dependencies = {} + + if os.path.isfile(state_file): + self.read_state_file() + else: + self.write_state_file() + + def update_config(self): + cfg.read_config() + + if self.port != cfg.controller_port: + self.port = cfg.controller_port + + # Loopback interface does not support multicast messaging, therefore + # revert to using unicast messaging when configured against the + # loopback device + if cfg.get_mgmt_iface() == constants.LOOPBACK_INTERFACE_NAME: + mgmt_ip = cfg.get_mgmt_ip() + self.mcast_addr = None + self.controller_address = mgmt_ip + self.agent_address = mgmt_ip + else: + self.mcast_addr = cfg.controller_mcast_group + self.controller_address = cfg.controller_mcast_group + self.agent_address = cfg.agent_mcast_group + + def socket_lock_acquire(self): + self.socket_lock.acquire() + + def socket_lock_release(self): + try: + self.socket_lock.release() + except Exception: + pass + + def write_state_file(self): + config = configparser.ConfigParser(strict=False) + + cfgfile = open(state_file, 'w') + + config.add_section('runtime') + config.set('runtime', 'patch_op_counter', str(self.patch_op_counter)) + config.write(cfgfile) + cfgfile.close() + + def read_state_file(self): + config = configparser.ConfigParser(strict=False) + + config.read(state_file) + + try: + counter = config.getint('runtime', 'patch_op_counter') + self.patch_op_counter = counter + + LOG.info("patch_op_counter is: %d", self.patch_op_counter) + except configparser.Error: + LOG.exception("Failed to read state info") + + def handle_nbr_patch_op_counter(self, host, nbr_patch_op_counter): + if self.patch_op_counter >= nbr_patch_op_counter: + return + + self.sync_from_nbr(host) + + def sync_from_nbr(self, host): + # Sync the software repo + host_url = utils.ip_to_url(host) + try: + output = subprocess.check_output(["rsync", + "-acv", + "--delete", + "--exclude", "tmp", + "rsync://%s/software/" % host_url, + "%s/" % patch_dir], + stderr=subprocess.STDOUT) + LOG.info("Synced to mate software via rsync: %s", output) + except subprocess.CalledProcessError as e: + LOG.error("Failed to rsync: %s", e.output) + return False + + try: + output = subprocess.check_output(["rsync", + "-acv", + "--delete", + "rsync://%s/repo/" % host_url, + "%s/" % repo_root_dir], + stderr=subprocess.STDOUT) + LOG.info("Synced to mate repo via rsync: %s", output) + except subprocess.CalledProcessError: + LOG.error("Failed to rsync: %s", output) + return False + + try: + for neighbour in list(self.hosts): + if (self.hosts[neighbour].nodetype == "controller" and + self.hosts[neighbour].ip == host): + LOG.info("Starting feed sync") + # The output is a string that lists the directories + # Example output: + # >>> dir_names = sh.ls("/var/www/pages/feed/") + # >>> dir_names.stdout + # b'rel-22.12 rel-22.5\n' + dir_names = sh.ls(constants.FEED_OSTREE_BASE_DIR) + + # Convert the output above into a list that can be iterated + # >>> list_of_dirs = dir_names.stdout.decode().rstrip().split() + # >>> print(list_of_dirs) + # ['rel-22.12', 'rel-22.5'] + + list_of_dirs = dir_names.stdout.decode("utf-8").rstrip().split() + + for rel_dir in list_of_dirs: + feed_ostree = "%s/%s/ostree_repo/" % (constants.FEED_OSTREE_BASE_DIR, rel_dir) + if not os.path.isdir(feed_ostree): + LOG.info("Skipping feed dir %s", feed_ostree) + continue + LOG.info("Syncing %s", feed_ostree) + output = subprocess.check_output(["ostree", + "--repo=%s" % feed_ostree, + "pull", + "--depth=-1", + "--mirror", + "starlingx"], + stderr=subprocess.STDOUT) + output = subprocess.check_output(["ostree", + "summary", + "--update", + "--repo=%s" % feed_ostree], + stderr=subprocess.STDOUT) + LOG.info("Synced to mate feed via ostree pull: %s", output) + except subprocess.CalledProcessError: + LOG.error("Failed to sync feed repo between controllers: %s", output) + return False + + self.read_state_file() + + self.patch_data_lock.acquire() + self.hosts_lock.acquire() + self.interim_state = {} + self.patch_data.load_all() + self.check_patch_states() + self.hosts_lock.release() + + if os.path.exists(app_dependency_filename): + try: + with open(app_dependency_filename, 'r') as f: + self.app_dependencies = json.loads(f.read()) + except Exception: + LOG.exception("Failed to read app dependencies: %s", app_dependency_filename) + else: + self.app_dependencies = {} + + self.patch_data_lock.release() + + return True + + def inc_patch_op_counter(self): + self.patch_op_counter += 1 + self.write_state_file() + + def check_patch_states(self): + # If we have no hosts, we can't be sure of the current patch state + if len(self.hosts) == 0: + for patch_id in self.patch_data.metadata: + self.patch_data.metadata[patch_id]["patchstate"] = constants.UNKNOWN + return + + # Default to allowing in-service patching + self.allow_insvc_patching = True + + # Take the detailed query results from the hosts and merge with the patch data + + self.hosts_lock.acquire() + + # Initialize patch state data based on repo state and interim_state presence + for patch_id in self.patch_data.metadata: + if patch_id in self.interim_state: + if self.patch_data.metadata[patch_id]["repostate"] == constants.AVAILABLE: + self.patch_data.metadata[patch_id]["patchstate"] = constants.PARTIAL_REMOVE + elif self.patch_data.metadata[patch_id]["repostate"] == constants.APPLIED: + self.patch_data.metadata[patch_id]["patchstate"] = constants.PARTIAL_APPLY + if self.patch_data.metadata[patch_id].get("reboot_required") != "N": + self.allow_insvc_patching = False + else: + self.patch_data.metadata[patch_id]["patchstate"] = \ + self.patch_data.metadata[patch_id]["repostate"] + + for ip in (ip for ip in list(self.hosts) if self.hosts[ip].out_of_date): + # If a host is out-of-date, the patch repostate is APPLIED and the patch's first + # commit doesn't match the active sysroot commit on the host, then change + # patchstate to PARTIAL-APPLY. + # If a host is out-of-date, the patch repostate is AVAILABLE and the patch's first + # commit is equal to the active sysroot commit on the host, then change the + # patchstate to PARTIAL-REMOVE. Additionally, change the patchstates of the + # patch required (directly or a chain dependency) by the current patch. + skip_patch = [] + for patch_id in self.patch_data.metadata: + # If the patch is on a different release than the host, skip it. + if self.patch_data.metadata[patch_id]["sw_version"] != self.hosts[ip].sw_version: + continue + + if patch_id not in skip_patch: + if self.patch_data.metadata[patch_id]["repostate"] == constants.AVAILABLE and \ + self.hosts[ip].latest_sysroot_commit == \ + self.patch_data.contents[patch_id]["commit1"]["commit"]: + self.patch_data.metadata[patch_id]["patchstate"] = constants.PARTIAL_REMOVE + patch_dependency_list = self.get_patch_dependency_list(patch_id) + for req_patch in patch_dependency_list: + if self.patch_data.metadata[req_patch]["repostate"] == constants.AVAILABLE: + self.patch_data.metadata[req_patch]["patchstate"] = constants.PARTIAL_REMOVE + else: + self.patch_data.metadata[req_patch]["patchstate"] = constants.APPLIED + skip_patch.append(req_patch) + elif self.patch_data.metadata[patch_id]["repostate"] == constants.APPLIED and \ + self.hosts[ip].latest_sysroot_commit != \ + self.patch_data.contents[patch_id]["commit1"]["commit"]: + self.patch_data.metadata[patch_id]["patchstate"] = constants.PARTIAL_APPLY + if self.patch_data.metadata[patch_id].get("reboot_required") != "N" and \ + (self.patch_data.metadata[patch_id]["patchstate"] == constants.PARTIAL_APPLY or + self.patch_data.metadata[patch_id]["patchstate"] == constants.PARTIAL_REMOVE): + self.allow_insvc_patching = False + + self.hosts_lock.release() + + def get_patch_dependency_list(self, patch_id): + """ + Returns a list of patch IDs that are required by this patch. + Example: If patch3 requires patch2 and patch2 requires patch1, + then this patch will return ['patch2', 'patch1'] for + input param patch_id='patch3' + :param patch_id: The patch ID + """ + if not self.patch_data.metadata[patch_id]["requires"]: + return [] + else: + patch_dependency_list = [] + for req_patch in self.patch_data.metadata[patch_id]["requires"]: + patch_dependency_list.append(req_patch) + patch_dependency_list = patch_dependency_list + self.get_patch_dependency_list(req_patch) + return patch_dependency_list + + def get_ostree_tar_filename(self, patch_sw_version, patch_id): + ''' + Returns the path of the ostree tarball + :param patch_sw_version: sw version this patch must be applied to + :param patch_id: The patch ID + ''' + ostree_tar_dir = package_dir[patch_sw_version] + ostree_tar_filename = "%s/%s-software.tar" % (ostree_tar_dir, patch_id) + return ostree_tar_filename + + def delete_restart_script(self, patch_id): + ''' + Deletes the restart script (if any) associated with the patch + :param patch_id: The patch ID + ''' + if not self.patch_data.metadata[patch_id].get("restart_script"): + return + + restart_script_path = "%s/%s" % (root_scripts_dir, self.patch_data.metadata[patch_id]["restart_script"]) + try: + # Delete the metadata + os.remove(restart_script_path) + except OSError: + msg = "Failed to remove restart script for %s" % patch_id + LOG.exception(msg) + raise PatchError(msg) + + def run_semantic_check(self, action, patch_list): + if not os.path.exists(INITIAL_CONFIG_COMPLETE_FLAG): + # Skip semantic checks if initial configuration isn't complete + return + + # Pass the current patch state to the semantic check as a series of args + patch_state_args = [] + for patch_id in list(self.patch_data.metadata): + patch_state = '%s=%s' % (patch_id, self.patch_data.metadata[patch_id]["patchstate"]) + patch_state_args += ['-p', patch_state] + + # Run semantic checks, if any + for patch_id in patch_list: + semchk = os.path.join(semantics_dir, action, patch_id) + + if os.path.exists(semchk): + try: + LOG.info("Running semantic check: %s", semchk) + subprocess.check_output([semchk] + patch_state_args, + stderr=subprocess.STDOUT) + LOG.info("Semantic check %s passed", semchk) + except subprocess.CalledProcessError as e: + msg = "Semantic check failed for %s:\n%s" % (patch_id, e.output) + LOG.exception(msg) + raise PatchFail(msg) + + def patch_import_api(self, patches): + """ + Import patches + :return: + """ + msg_info = "" + msg_warning = "" + msg_error = "" + + # Refresh data, if needed + self.base_pkgdata.loaddirs() + + # Protect against duplications + patch_list = sorted(list(set(patches))) + + # First, make sure the specified files exist + for patch in patch_list: + if not os.path.isfile(patch): + raise PatchFail("File does not exist: %s" % patch) + + try: + if not os.path.exists(avail_dir): + os.makedirs(avail_dir) + if not os.path.exists(applied_dir): + os.makedirs(applied_dir) + if not os.path.exists(committed_dir): + os.makedirs(committed_dir) + except os.error: + msg = "Failed to create directories" + LOG.exception(msg) + raise PatchFail(msg) + + msg = "Importing patches: %s" % ",".join(patch_list) + LOG.info(msg) + audit_log_info(msg) + + for patch in patch_list: + msg = "Importing patch: %s" % patch + LOG.info(msg) + audit_log_info(msg) + + # Get the patch_id from the filename + # and check to see if it's already imported + (patch_id, ext) = os.path.splitext(os.path.basename(patch)) + if patch_id in self.patch_data.metadata: + if self.patch_data.metadata[patch_id]["repostate"] == constants.APPLIED: + mdir = applied_dir + elif self.patch_data.metadata[patch_id]["repostate"] == constants.COMMITTED: + msg = "%s is committed. Metadata not updated" % patch_id + LOG.info(msg) + msg_info += msg + "\n" + continue + else: + mdir = avail_dir + + try: + thispatch = PatchFile.extract_patch(patch, + metadata_dir=mdir, + metadata_only=True, + existing_content=self.patch_data.contents[patch_id], + base_pkgdata=self.base_pkgdata) + self.patch_data.update_patch(thispatch) + msg = "%s is already imported. Updated metadata only" % patch_id + LOG.info(msg) + msg_info += msg + "\n" + except PatchMismatchFailure: + msg = "Contents of %s do not match re-imported patch" % patch_id + LOG.exception(msg) + msg_error += msg + "\n" + continue + except PatchValidationFailure as e: + msg = "Patch validation failed for %s" % patch_id + if str(e) is not None and str(e) != '': + msg += ":\n%s" % str(e) + LOG.exception(msg) + msg_error += msg + "\n" + continue + except PatchFail: + msg = "Failed to import patch %s" % patch_id + LOG.exception(msg) + msg_error += msg + "\n" + + continue + + if ext != ".patch": + msg = "File must end in .patch extension: %s" \ + % os.path.basename(patch) + LOG.exception(msg) + msg_error += msg + "\n" + continue + + try: + thispatch = PatchFile.extract_patch(patch, + metadata_dir=avail_dir, + base_pkgdata=self.base_pkgdata) + + msg_info += "%s is now available\n" % patch_id + self.patch_data.add_patch(thispatch) + + self.patch_data.metadata[patch_id]["repostate"] = constants.AVAILABLE + if len(self.hosts) > 0: + self.patch_data.metadata[patch_id]["patchstate"] = constants.AVAILABLE + else: + self.patch_data.metadata[patch_id]["patchstate"] = constants.UNKNOWN + except PatchValidationFailure as e: + msg = "Patch validation failed for %s" % patch_id + if str(e) is not None and str(e) != '': + msg += ":\n%s" % str(e) + LOG.exception(msg) + msg_error += msg + "\n" + continue + except PatchFail: + msg = "Failed to import patch %s" % patch_id + LOG.exception(msg) + msg_error += msg + "\n" + continue + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + def patch_apply_api(self, patch_ids, **kwargs): + """ + Apply patches, moving patches from available to applied and updating repo + :return: + """ + msg_info = "" + msg_warning = "" + msg_error = "" + + # Protect against duplications + patch_list = sorted(list(set(patch_ids))) + + msg = "Applying patches: %s" % ",".join(patch_list) + LOG.info(msg) + audit_log_info(msg) + + if "--all" in patch_list: + # Set patch_ids to list of all available patches + # We're getting this list now, before we load the applied patches + patch_list = [] + for patch_id in sorted(list(self.patch_data.metadata)): + if self.patch_data.metadata[patch_id]["repostate"] == constants.AVAILABLE: + patch_list.append(patch_id) + + if len(patch_list) == 0: + msg_info += "There are no available patches to be applied.\n" + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + # First, verify that all specified patches exist + id_verification = True + for patch_id in patch_list: + if patch_id not in self.patch_data.metadata: + msg = "Patch %s does not exist" % patch_id + LOG.error(msg) + msg_error += msg + "\n" + id_verification = False + + if not id_verification: + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + # Order patches such that + # If P2 requires P1 + # P3 requires P2 + # P4 requires P3 + # Apply order: [P1, P2, P3, P4] + # Patch with lowest dependency gets applied first. + patch_list = self.patch_apply_remove_order(patch_list, reverse=True) + + msg = "Patch Apply order: %s" % ",".join(patch_list) + LOG.info(msg) + audit_log_info(msg) + + # Check for patches that can't be applied during an upgrade + upgrade_check = True + for patch_id in patch_list: + if self.patch_data.metadata[patch_id]["sw_version"] != SW_VERSION \ + and self.patch_data.metadata[patch_id].get("apply_active_release_only") == "Y": + msg = "%s cannot be applied in an upgrade" % patch_id + LOG.error(msg) + msg_error += msg + "\n" + upgrade_check = False + + if not upgrade_check: + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + # Next, check the patch dependencies + # required_patches will map the required patch to the patches that need it + required_patches = {} + for patch_id in patch_list: + for req_patch in self.patch_data.metadata[patch_id]["requires"]: + # Ignore patches in the op set + if req_patch in patch_list: + continue + + if req_patch not in required_patches: + required_patches[req_patch] = [] + + required_patches[req_patch].append(patch_id) + + # Now verify the state of the required patches + req_verification = True + for req_patch, iter_patch_list in required_patches.items(): + if req_patch not in self.patch_data.metadata \ + or self.patch_data.metadata[req_patch]["repostate"] == constants.AVAILABLE: + msg = "%s is required by: %s" % (req_patch, ", ".join(sorted(iter_patch_list))) + msg_error += msg + "\n" + LOG.info(msg) + req_verification = False + + if not req_verification: + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + if kwargs.get("skip-semantic") != "yes": + self.run_semantic_check(constants.SEMANTIC_PREAPPLY, patch_list) + + # Start applying the patches + for patch_id in patch_list: + msg = "Applying patch: %s" % patch_id + LOG.info(msg) + audit_log_info(msg) + + if self.patch_data.metadata[patch_id]["repostate"] == constants.APPLIED \ + or self.patch_data.metadata[patch_id]["repostate"] == constants.COMMITTED: + msg = "%s is already in the repo" % patch_id + LOG.info(msg) + msg_info += msg + "\n" + continue + + patch_sw_version = self.patch_data.metadata[patch_id]["sw_version"] + + # STX R7.0 is the first version to support ostree + # earlier formats will not have "base" and are unsupported + if self.patch_data.contents[patch_id].get("base") is None: + msg = "%s is an unsupported patch format" % patch_id + LOG.info(msg) + msg_info += msg + "\n" + continue + + latest_commit = "" + try: + latest_commit = ostree_utils.get_feed_latest_commit(patch_sw_version) + except OSTreeCommandFail: + LOG.exception("Failure during commit consistency check for %s.", patch_id) + + if self.patch_data.contents[patch_id]["base"]["commit"] != latest_commit: + msg = "The base commit %s for %s does not match the latest commit %s " \ + "on this system." \ + % (self.patch_data.contents[patch_id]["base"]["commit"], + patch_id, + latest_commit) + LOG.info(msg) + msg_info += msg + "\n" + continue + + ostree_tar_filename = self.get_ostree_tar_filename(patch_sw_version, patch_id) + + # Create a temporary working directory + tmpdir = tempfile.mkdtemp(prefix="patch_") + + # Save the current directory, so we can chdir back after + orig_wd = os.getcwd() + + # Change to the tmpdir + os.chdir(tmpdir) + + try: + # Extract the software.tar + tar = tarfile.open(ostree_tar_filename) + tar.extractall() + feed_ostree = "%s/rel-%s/ostree_repo" % (constants.FEED_OSTREE_BASE_DIR, patch_sw_version) + # Copy extracted folders of software.tar to the feed ostree repo + shutil.copytree(tmpdir, feed_ostree, dirs_exist_ok=True) + except tarfile.TarError: + msg = "Failed to extract the ostree tarball for %s" % patch_id + LOG.exception(msg) + raise OSTreeTarFail(msg) + except shutil.Error: + msg = "Failed to copy the ostree tarball for %s" % patch_id + LOG.exception(msg) + raise OSTreeTarFail(msg) + finally: + # Change back to original working dir + os.chdir(orig_wd) + shutil.rmtree(tmpdir, ignore_errors=True) + + try: + # Move the metadata from avail to applied dir + shutil.move("%s/%s-metadata.xml" % (avail_dir, patch_id), + "%s/%s-metadata.xml" % (applied_dir, patch_id)) + + msg_info += "%s is now in the repo\n" % patch_id + except shutil.Error: + msg = "Failed to move the metadata for %s" % patch_id + LOG.exception(msg) + raise MetadataFail(msg) + + self.patch_data.metadata[patch_id]["repostate"] = constants.APPLIED + if len(self.hosts) > 0: + self.patch_data.metadata[patch_id]["patchstate"] = constants.PARTIAL_APPLY + else: + self.patch_data.metadata[patch_id]["patchstate"] = constants.UNKNOWN + + # Commit1 in patch metadata.xml file represents the latest commit + # after this patch has been applied to the feed repo + self.latest_feed_commit = self.patch_data.contents[patch_id]["commit1"]["commit"] + + self.hosts_lock.acquire() + self.interim_state[patch_id] = list(self.hosts) + self.hosts_lock.release() + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + def patch_apply_remove_order(self, patch_ids, reverse=False): + # Protect against duplications + patch_list = sorted(list(set(patch_ids))) + + # single patch + if len(patch_list) == 1: + return patch_list + + # versions of patches in the list don't match + ver = None + for patch_id in patch_list: + if ver is None: + ver = self.patch_data.metadata[patch_id]["sw_version"] + elif self.patch_data.metadata[patch_id]["sw_version"] != ver: + return None + + # Multiple patches with require dependencies + highest_dependency = 0 + patch_remove_order = None + patch_with_highest_dependency = None + + for patch_id in patch_list: + dependency_list = self.get_patch_dependency_list(patch_id) + if len(dependency_list) > highest_dependency: + highest_dependency = len(dependency_list) + patch_with_highest_dependency = patch_id + patch_remove_order = dependency_list + + patch_list = [patch_with_highest_dependency] + patch_remove_order + if reverse: + patch_list.reverse() + return patch_list + + def patch_remove_api(self, patch_ids, **kwargs): + """ + Remove patches, moving patches from applied to available and updating repo + :return: + """ + msg_info = "" + msg_warning = "" + msg_error = "" + remove_unremovable = False + + # First, verify that all specified patches exist + id_verification = True + for patch_id in sorted(list(set(patch_ids))): + if patch_id not in self.patch_data.metadata: + msg = "Patch %s does not exist" % patch_id + LOG.error(msg) + msg_error += msg + "\n" + id_verification = False + + if not id_verification: + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + patch_list = self.patch_apply_remove_order(patch_ids) + + if patch_list is None: + msg = "Patch list provided belongs to different software versions." + LOG.error(msg) + msg_error += msg + "\n" + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + msg = "Removing patches: %s" % ",".join(patch_list) + LOG.info(msg) + audit_log_info(msg) + + if kwargs.get("removeunremovable") == "yes": + remove_unremovable = True + + # See if any of the patches are marked as unremovable + unremovable_verification = True + for patch_id in patch_list: + if self.patch_data.metadata[patch_id].get("unremovable") == "Y": + if remove_unremovable: + msg = "Unremovable patch %s being removed" % patch_id + LOG.warning(msg) + msg_warning += msg + "\n" + else: + msg = "Patch %s is not removable" % patch_id + LOG.error(msg) + msg_error += msg + "\n" + unremovable_verification = False + elif self.patch_data.metadata[patch_id]['repostate'] == constants.COMMITTED: + msg = "Patch %s is committed and cannot be removed" % patch_id + LOG.error(msg) + msg_error += msg + "\n" + unremovable_verification = False + + if not unremovable_verification: + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + # Next, see if any of the patches are required by applied patches + # required_patches will map the required patch to the patches that need it + required_patches = {} + for patch_iter in list(self.patch_data.metadata): + # Ignore patches in the op set + if patch_iter in patch_list: + continue + + # Only check applied patches + if self.patch_data.metadata[patch_iter]["repostate"] == constants.AVAILABLE: + continue + + for req_patch in self.patch_data.metadata[patch_iter]["requires"]: + if req_patch not in patch_list: + continue + + if req_patch not in required_patches: + required_patches[req_patch] = [] + + required_patches[req_patch].append(patch_iter) + + if len(required_patches) > 0: + for req_patch, iter_patch_list in required_patches.items(): + msg = "%s is required by: %s" % (req_patch, ", ".join(sorted(iter_patch_list))) + msg_error += msg + "\n" + LOG.info(msg) + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + if kwargs.get("skipappcheck") != "yes": + # Check application dependencies before removing + required_patches = {} + for patch_id in patch_list: + for appname, iter_patch_list in self.app_dependencies.items(): + if patch_id in iter_patch_list: + if patch_id not in required_patches: + required_patches[patch_id] = [] + required_patches[patch_id].append(appname) + + if len(required_patches) > 0: + for req_patch, app_list in required_patches.items(): + msg = "%s is required by application(s): %s" % (req_patch, ", ".join(sorted(app_list))) + msg_error += msg + "\n" + LOG.info(msg) + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + if kwargs.get("skip-semantic") != "yes": + self.run_semantic_check(constants.SEMANTIC_PREREMOVE, patch_list) + + for patch_id in patch_list: + msg = "Removing patch: %s" % patch_id + LOG.info(msg) + audit_log_info(msg) + + if self.patch_data.metadata[patch_id]["repostate"] == constants.AVAILABLE: + msg = "%s is not in the repo" % patch_id + LOG.info(msg) + msg_info += msg + "\n" + continue + + patch_sw_version = self.patch_data.metadata[patch_id]["sw_version"] + # 22.12 is the first version to support ostree + # earlier formats will not have "base" and are unsupported + # simply move them to 'available and skip to the next patch + if self.patch_data.contents[patch_id].get("base") is None: + msg = "%s is an unsupported patch format" % patch_id + LOG.info(msg) + msg_info += msg + "\n" + + else: + # this is an ostree patch + # Base commit is fetched from the patch metadata + base_commit = self.patch_data.contents[patch_id]["base"]["commit"] + feed_ostree = "%s/rel-%s/ostree_repo" % (constants.FEED_OSTREE_BASE_DIR, patch_sw_version) + try: + # Reset the ostree HEAD + ostree_utils.reset_ostree_repo_head(base_commit, feed_ostree) + + # Delete all commits that belong to this patch + for i in range(int(self.patch_data.contents[patch_id]["number_of_commits"])): + commit_to_delete = self.patch_data.contents[patch_id]["commit%s" % (i + 1)]["commit"] + ostree_utils.delete_ostree_repo_commit(commit_to_delete, feed_ostree) + + # Update the feed ostree summary + ostree_utils.update_repo_summary_file(feed_ostree) + + except OSTreeCommandFail: + LOG.exception("Failure during patch remove for %s.", patch_id) + + # update metadata + try: + # Move the metadata to the available dir + shutil.move("%s/%s-metadata.xml" % (applied_dir, patch_id), + "%s/%s-metadata.xml" % (avail_dir, patch_id)) + msg_info += "%s has been removed from the repo\n" % patch_id + except shutil.Error: + msg = "Failed to move the metadata for %s" % patch_id + LOG.exception(msg) + raise MetadataFail(msg) + + # update patchstate and repostate + self.patch_data.metadata[patch_id]["repostate"] = constants.AVAILABLE + if len(self.hosts) > 0: + self.patch_data.metadata[patch_id]["patchstate"] = constants.PARTIAL_REMOVE + else: + self.patch_data.metadata[patch_id]["patchstate"] = constants.UNKNOWN + + # only update lastest_feed_commit if it is an ostree patch + if self.patch_data.contents[patch_id].get("base") is not None: + # Base Commit in patch metadata.xml file represents the latest commit + # after this patch has been removed from the feed repo + self.latest_feed_commit = self.patch_data.contents[patch_id]["base"]["commit"] + + self.hosts_lock.acquire() + self.interim_state[patch_id] = list(self.hosts) + self.hosts_lock.release() + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + def patch_delete_api(self, patch_ids): + """ + Delete patches + :return: + """ + msg_info = "" + msg_warning = "" + msg_error = "" + + # Protect against duplications + patch_list = sorted(list(set(patch_ids))) + + msg = "Deleting patches: %s" % ",".join(patch_list) + LOG.info(msg) + audit_log_info(msg) + + # Verify patches exist and are in proper state first + id_verification = True + for patch_id in patch_list: + if patch_id not in self.patch_data.metadata: + msg = "Patch %s does not exist" % patch_id + LOG.error(msg) + msg_error += msg + "\n" + id_verification = False + continue + + # Get the aggregated patch state, if possible + patchstate = constants.UNKNOWN + if patch_id in self.patch_data.metadata: + patchstate = self.patch_data.metadata[patch_id]["patchstate"] + + if self.patch_data.metadata[patch_id]["repostate"] != constants.AVAILABLE or \ + (patchstate != constants.AVAILABLE and patchstate != constants.UNKNOWN): + msg = "Patch %s not in Available state" % patch_id + LOG.error(msg) + msg_error += msg + "\n" + id_verification = False + continue + + if not id_verification: + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + # Handle operation + for patch_id in patch_list: + patch_sw_version = self.patch_data.metadata[patch_id]["sw_version"] + + # Need to support delete of older centos patches (metadata) from upgrades. + + # Delete ostree content if it exists. + # RPM based patches (from upgrades) will not have ostree contents + ostree_tar_filename = self.get_ostree_tar_filename(patch_sw_version, patch_id) + if os.path.isfile(ostree_tar_filename): + try: + os.remove(ostree_tar_filename) + except OSError: + msg = "Failed to remove ostree tarball %s" % ostree_tar_filename + LOG.exception(msg) + raise OSTreeTarFail(msg) + + try: + # Delete the metadata + os.remove("%s/%s-metadata.xml" % (avail_dir, patch_id)) + except OSError: + msg = "Failed to remove metadata for %s" % patch_id + LOG.exception(msg) + raise MetadataFail(msg) + + self.delete_restart_script(patch_id) + self.patch_data.delete_patch(patch_id) + msg = "%s has been deleted" % patch_id + LOG.info(msg) + msg_info += msg + "\n" + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + def patch_init_release_api(self, release): + """ + Create an empty repo for a new release + :return: + """ + msg_info = "" + msg_warning = "" + msg_error = "" + + msg = "Initializing repo for: %s" % release + LOG.info(msg) + audit_log_info(msg) + + if release == SW_VERSION: + msg = "Rejected: Requested release %s is running release" % release + msg_error += msg + "\n" + LOG.info(msg) + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + # Refresh data + self.base_pkgdata.loaddirs() + + self.patch_data.load_all_metadata(avail_dir, repostate=constants.AVAILABLE) + self.patch_data.load_all_metadata(applied_dir, repostate=constants.APPLIED) + self.patch_data.load_all_metadata(committed_dir, repostate=constants.COMMITTED) + + repo_dir[release] = "%s/rel-%s" % (repo_root_dir, release) + + # Verify the release doesn't already exist + if os.path.exists(repo_dir[release]): + msg = "Patch repository for %s already exists" % release + msg_info += msg + "\n" + LOG.info(msg) + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + # Create the repo + try: + # todo(jcasteli) determine if ostree change needs a createrepo equivalent + output = "UNDER CONSTRUCTION for OSTREE" + LOG.info("Repo[%s] updated:\n%s", release, output) + except Exception: + msg = "Failed to update the repo for %s" % release + LOG.exception(msg) + + # Wipe out what was created + shutil.rmtree(repo_dir[release]) + del repo_dir[release] + + raise PatchFail(msg) + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + def patch_del_release_api(self, release): + """ + Delete the repo and patches for second release + :return: + """ + msg_info = "" + msg_warning = "" + msg_error = "" + + msg = "Deleting repo and patches for: %s" % release + LOG.info(msg) + audit_log_info(msg) + + if release == SW_VERSION: + msg = "Rejected: Requested release %s is running release" % release + msg_error += msg + "\n" + LOG.info(msg) + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + # Delete patch XML files + for patch_id in list(self.patch_data.metadata): + if self.patch_data.metadata[patch_id]["sw_version"] != release: + continue + + if self.patch_data.metadata[patch_id]["repostate"] == constants.APPLIED: + mdir = applied_dir + elif self.patch_data.metadata[patch_id]["repostate"] == constants.COMMITTED: + mdir = committed_dir + else: + mdir = avail_dir + + for action in constants.SEMANTIC_ACTIONS: + action_file = os.path.join(semantics_dir, action, patch_id) + if not os.path.isfile(action_file): + continue + + try: + os.remove(action_file) + except OSError: + msg = "Failed to remove semantic %s" % action_file + LOG.exception(msg) + raise SemanticFail(msg) + + try: + # Delete the metadata + os.remove("%s/%s-metadata.xml" % (mdir, patch_id)) + except OSError: + msg = "Failed to remove metadata for %s" % patch_id + LOG.exception(msg) + + # Refresh patch data + self.patch_data = PatchData() + self.patch_data.load_all_metadata(avail_dir, repostate=constants.AVAILABLE) + self.patch_data.load_all_metadata(applied_dir, repostate=constants.APPLIED) + self.patch_data.load_all_metadata(committed_dir, repostate=constants.COMMITTED) + + raise MetadataFail(msg) + + # Delete the packages dir + package_dir[release] = "%s/%s" % (root_package_dir, release) + if os.path.exists(package_dir[release]): + try: + shutil.rmtree(package_dir[release]) + except shutil.Error: + msg = "Failed to delete package dir for %s" % release + LOG.exception(msg) + + del package_dir[release] + + # Verify the release exists + repo_dir[release] = "%s/rel-%s" % (repo_root_dir, release) + if not os.path.exists(repo_dir[release]): + # Nothing to do + msg = "Patch repository for %s does not exist" % release + msg_info += msg + "\n" + LOG.info(msg) + del repo_dir[release] + + # Refresh patch data + self.patch_data = PatchData() + self.patch_data.load_all_metadata(avail_dir, repostate=constants.AVAILABLE) + self.patch_data.load_all_metadata(applied_dir, repostate=constants.APPLIED) + self.patch_data.load_all_metadata(committed_dir, repostate=constants.COMMITTED) + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + # Delete the repo + try: + shutil.rmtree(repo_dir[release]) + except shutil.Error: + msg = "Failed to delete repo for %s" % release + LOG.exception(msg) + + del repo_dir[release] + + if self.base_pkgdata is not None and release in self.base_pkgdata.pkgs: + del self.base_pkgdata.pkgs[release] + + # Refresh patch data + self.patch_data = PatchData() + self.patch_data.load_all_metadata(avail_dir, repostate=constants.AVAILABLE) + self.patch_data.load_all_metadata(applied_dir, repostate=constants.APPLIED) + self.patch_data.load_all_metadata(committed_dir, repostate=constants.COMMITTED) + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + def patch_query_what_requires(self, patch_ids): + """ + Query the known patches to see which have dependencies on the specified patches + :return: + """ + msg_info = "" + msg_warning = "" + msg_error = "" + + msg = "Querying what requires patches: %s" % ",".join(patch_ids) + LOG.info(msg) + audit_log_info(msg) + + # First, verify that all specified patches exist + id_verification = True + for patch_id in patch_ids: + if patch_id not in self.patch_data.metadata: + msg = "Patch %s does not exist" % patch_id + LOG.error(msg) + msg_error += msg + "\n" + id_verification = False + + if not id_verification: + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + required_patches = {} + for patch_iter in list(self.patch_data.metadata): + for req_patch in self.patch_data.metadata[patch_iter]["requires"]: + if req_patch not in patch_ids: + continue + + if req_patch not in required_patches: + required_patches[req_patch] = [] + + required_patches[req_patch].append(patch_iter) + + for patch_id in patch_ids: + if patch_id in required_patches: + iter_patch_list = required_patches[patch_id] + msg_info += "%s is required by: %s\n" % (patch_id, ", ".join(sorted(iter_patch_list))) + else: + msg_info += "%s is not required by any patches.\n" % patch_id + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + def send_latest_feed_commit_to_agent(self): + """ + Notify the patch agent that the latest commit on the feed + repo has been updated + """ + # Skip sending messages if host not yet provisioned + if self.sock_out is None: + LOG.info("Skipping send feed commit to agent") + return + + send_commit_to_agent = PatchMessageSendLatestFeedCommit() + self.socket_lock.acquire() + send_commit_to_agent.send(self.sock_out) + self.socket_lock.release() + + def patch_sync(self): + # Increment the patch_op_counter here + self.inc_patch_op_counter() + + self.patch_data_lock.acquire() + # self.patch_data.load_all() + self.check_patch_states() + self.patch_data_lock.release() + + if self.sock_out is None: + return True + + # Send the sync requests + + self.controller_neighbours_lock.acquire() + for n in self.controller_neighbours: + self.controller_neighbours[n].clear_synced() + self.controller_neighbours_lock.release() + + msg = PatchMessageSyncReq() + self.socket_lock.acquire() + msg.send(self.sock_out) + self.socket_lock.release() + + # Now we wait, up to two mins. future enhancement: Wait on a condition + my_ip = cfg.get_mgmt_ip() + sync_rc = False + max_time = time.time() + 120 + while time.time() < max_time: + all_done = True + self.controller_neighbours_lock.acquire() + for n in self.controller_neighbours: + if n != my_ip and not self.controller_neighbours[n].get_synced(): + all_done = False + self.controller_neighbours_lock.release() + + if all_done: + LOG.info("Sync complete") + sync_rc = True + break + + time.sleep(0.5) + + # Send hellos to the hosts now, to get queries performed + hello_agent = PatchMessageHelloAgent() + self.socket_lock.acquire() + hello_agent.send(self.sock_out) + self.socket_lock.release() + + if not sync_rc: + LOG.info("Timed out waiting for sync completion") + return sync_rc + + def patch_query_cached(self, **kwargs): + query_state = None + if "show" in kwargs: + if kwargs["show"] == "available": + query_state = constants.AVAILABLE + elif kwargs["show"] == "applied": + query_state = constants.APPLIED + elif kwargs["show"] == "committed": + query_state = constants.COMMITTED + + query_release = None + if "release" in kwargs: + query_release = kwargs["release"] + + results = {} + self.patch_data_lock.acquire() + if query_state is None and query_release is None: + # Return everything + results = self.patch_data.metadata + else: + # Filter results + for patch_id, data in self.patch_data.metadata.items(): + if query_state is not None and data["repostate"] != query_state: + continue + if query_release is not None and data["sw_version"] != query_release: + continue + results[patch_id] = data + self.patch_data_lock.release() + + return results + + def patch_query_specific_cached(self, patch_ids): + audit_log_info("Patch show") + + results = {"metadata": {}, + "contents": {}, + "error": ""} + + self.patch_data_lock.acquire() + + for patch_id in patch_ids: + if patch_id not in list(self.patch_data.metadata): + results["error"] += "%s is unrecognized\n" % patch_id + + for patch_id, data in self.patch_data.metadata.items(): + if patch_id in patch_ids: + results["metadata"][patch_id] = data + for patch_id, data in self.patch_data.contents.items(): + if patch_id in patch_ids: + results["contents"][patch_id] = data + + self.patch_data_lock.release() + + return results + + def get_dependencies(self, patch_ids, recursive): + dependencies = set() + patch_added = False + + self.patch_data_lock.acquire() + + # Add patches to workset + for patch_id in sorted(patch_ids): + dependencies.add(patch_id) + patch_added = True + + while patch_added: + patch_added = False + for patch_id in sorted(dependencies): + for req in self.patch_data.metadata[patch_id]["requires"]: + if req not in dependencies: + dependencies.add(req) + patch_added = recursive + + self.patch_data_lock.release() + + return sorted(dependencies) + + def patch_query_dependencies(self, patch_ids, **kwargs): + msg = "Patch query-dependencies %s" % patch_ids + LOG.info(msg) + audit_log_info(msg) + + failure = False + + results = {"patches": [], + "error": ""} + + recursive = False + if kwargs.get("recursive") == "yes": + recursive = True + + self.patch_data_lock.acquire() + + # Verify patch IDs + for patch_id in sorted(patch_ids): + if patch_id not in list(self.patch_data.metadata): + errormsg = "%s is unrecognized\n" % patch_id + LOG.info("patch_query_dependencies: %s", errormsg) + results["error"] += errormsg + failure = True + self.patch_data_lock.release() + + if failure: + LOG.info("patch_query_dependencies failed") + return results + + results["patches"] = self.get_dependencies(patch_ids, recursive) + + return results + + def patch_commit(self, patch_ids, dry_run=False): + msg = "Patch commit %s" % patch_ids + LOG.info(msg) + audit_log_info(msg) + + try: + if not os.path.exists(committed_dir): + os.makedirs(committed_dir) + except os.error: + msg = "Failed to create %s" % committed_dir + LOG.exception(msg) + raise PatchFail(msg) + + failure = False + recursive = True + cleanup_files = set() + results = {"info": "", + "error": ""} + + # Ensure there are only REL patches + non_rel_list = [] + self.patch_data_lock.acquire() + for patch_id in self.patch_data.metadata: + if self.patch_data.metadata[patch_id]['status'] != constants.STATUS_RELEASED: + non_rel_list.append(patch_id) + self.patch_data_lock.release() + + if len(non_rel_list) > 0: + errormsg = "A commit cannot be performed with non-REL status patches in the system:\n" + for patch_id in non_rel_list: + errormsg += " %s\n" % patch_id + LOG.info("patch_commit rejected: %s", errormsg) + results["error"] += errormsg + return results + + # Verify patch IDs + self.patch_data_lock.acquire() + for patch_id in sorted(patch_ids): + if patch_id not in list(self.patch_data.metadata): + errormsg = "%s is unrecognized\n" % patch_id + LOG.info("patch_commit: %s", errormsg) + results["error"] += errormsg + failure = True + self.patch_data_lock.release() + + if failure: + LOG.info("patch_commit: Failed patch ID check") + return results + + commit_list = self.get_dependencies(patch_ids, recursive) + + # Check patch states + avail_list = [] + self.patch_data_lock.acquire() + for patch_id in commit_list: + if self.patch_data.metadata[patch_id]['patchstate'] != constants.APPLIED \ + and self.patch_data.metadata[patch_id]['patchstate'] != constants.COMMITTED: + avail_list.append(patch_id) + self.patch_data_lock.release() + + if len(avail_list) > 0: + errormsg = "The following patches are not applied and cannot be committed:\n" + for patch_id in avail_list: + errormsg += " %s\n" % patch_id + LOG.info("patch_commit rejected: %s", errormsg) + results["error"] += errormsg + return results + + with self.patch_data_lock: + for patch_id in commit_list: + # Fetch file paths that need to be cleaned up to + # free patch storage disk space + if self.patch_data.metadata[patch_id].get("restart_script"): + restart_script_path = "%s/%s" % \ + (root_scripts_dir, + self.patch_data.metadata[patch_id]["restart_script"]) + if os.path.exists(restart_script_path): + cleanup_files.add(restart_script_path) + patch_sw_version = self.patch_data.metadata[patch_id]["sw_version"] + abs_ostree_tar_dir = package_dir[patch_sw_version] + software_tar_path = "%s/%s-software.tar" % (abs_ostree_tar_dir, patch_id) + if os.path.exists(software_tar_path): + cleanup_files.add(software_tar_path) + + # Calculate disk space + disk_space = 0 + for file in cleanup_files: + statinfo = os.stat(file) + disk_space += statinfo.st_size + + if dry_run: + results["info"] = "This commit operation would free %0.2f MiB" % (disk_space / (1024.0 * 1024.0)) + return results + + # Do the commit + + # Move the metadata to the committed dir + for patch_id in commit_list: + metadata_fname = "%s-metadata.xml" % patch_id + applied_fname = os.path.join(applied_dir, metadata_fname) + committed_fname = os.path.join(committed_dir, metadata_fname) + if os.path.exists(applied_fname): + try: + shutil.move(applied_fname, committed_fname) + except shutil.Error: + msg = "Failed to move the metadata for %s" % patch_id + LOG.exception(msg) + raise MetadataFail(msg) + + # Delete the files + for file in cleanup_files: + try: + os.remove(file) + except OSError: + msg = "Failed to remove: %s" % file + LOG.exception(msg) + raise MetadataFail(msg) + + self.patch_data.load_all() + + results["info"] = "The patches have been committed." + return results + + def query_host_cache(self): + output = [] + + self.hosts_lock.acquire() + for nbr in list(self.hosts): + host = self.hosts[nbr].get_dict() + host["interim_state"] = False + for patch_id in list(pc.interim_state): + if nbr in pc.interim_state[patch_id]: + host["interim_state"] = True + + output.append(host) + + self.hosts_lock.release() + + return output + + def any_patch_host_installing(self): + rc = False + + self.hosts_lock.acquire() + for host in self.hosts.values(): + if host.state == constants.PATCH_AGENT_STATE_INSTALLING: + rc = True + break + + self.hosts_lock.release() + + return rc + + def copy_restart_scripts(self): + with self.patch_data_lock: + for patch_id in self.patch_data.metadata: + if (self.patch_data.metadata[patch_id]["patchstate"] in + [constants.PARTIAL_APPLY, constants.PARTIAL_REMOVE]) \ + and self.patch_data.metadata[patch_id].get("restart_script"): + try: + restart_script_name = self.patch_data.metadata[patch_id]["restart_script"] + restart_script_path = "%s/%s" \ + % (root_scripts_dir, restart_script_name) + dest_path = constants.PATCH_SCRIPTS_STAGING_DIR + dest_script_file = "%s/%s" \ + % (constants.PATCH_SCRIPTS_STAGING_DIR, restart_script_name) + if not os.path.exists(dest_path): + os.makedirs(dest_path, 0o700) + shutil.copyfile(restart_script_path, dest_script_file) + os.chmod(dest_script_file, 0o700) + msg = "Creating restart script for %s" % patch_id + LOG.info(msg) + except shutil.Error: + msg = "Failed to copy the restart script for %s" % patch_id + LOG.exception(msg) + raise PatchError(msg) + elif self.patch_data.metadata[patch_id].get("restart_script"): + try: + restart_script_name = self.patch_data.metadata[patch_id]["restart_script"] + restart_script_path = "%s/%s" \ + % (constants.PATCH_SCRIPTS_STAGING_DIR, restart_script_name) + if os.path.exists(restart_script_path): + os.remove(restart_script_path) + msg = "Removing restart script for %s" % patch_id + LOG.info(msg) + except shutil.Error: + msg = "Failed to delete the restart script for %s" % patch_id + LOG.exception(msg) + + def patch_host_install(self, host_ip, force, async_req=False): + msg_info = "" + msg_warning = "" + msg_error = "" + + ip = host_ip + + self.hosts_lock.acquire() + # If not in hosts table, maybe a hostname was used instead + if host_ip not in self.hosts: + try: + ip = utils.gethostbyname(host_ip) + if ip not in self.hosts: + # Translated successfully, but IP isn't in the table. + # Raise an exception to drop out to the failure handling + raise PatchError("Host IP (%s) not in table" % ip) + except Exception: + self.hosts_lock.release() + msg = "Unknown host specified: %s" % host_ip + msg_error += msg + "\n" + LOG.error("Error in host-install: %s", msg) + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + msg = "Running host-install for %s (%s), force=%s, async_req=%s" % (host_ip, ip, force, async_req) + LOG.info(msg) + audit_log_info(msg) + + if self.allow_insvc_patching: + LOG.info("Allowing in-service patching") + force = True + self.copy_restart_scripts() + + self.hosts[ip].install_pending = True + self.hosts[ip].install_status = False + self.hosts[ip].install_reject_reason = None + self.hosts_lock.release() + + installreq = PatchMessageAgentInstallReq() + installreq.ip = ip + installreq.force = force + installreq.encode() + self.socket_lock.acquire() + installreq.send(self.sock_out) + self.socket_lock.release() + + if async_req: + # async_req install requested, so return now + msg = "Patch installation request sent to %s." % self.hosts[ip].hostname + msg_info += msg + "\n" + LOG.info("host-install async_req: %s", msg) + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + # Now we wait, up to ten mins. future enhancement: Wait on a condition + resp_rx = False + max_time = time.time() + 600 + while time.time() < max_time: + self.hosts_lock.acquire() + if ip not in self.hosts: + # The host aged out while we were waiting + self.hosts_lock.release() + msg = "Agent expired while waiting: %s" % ip + msg_error += msg + "\n" + LOG.error("Error in host-install: %s", msg) + break + + if not self.hosts[ip].install_pending: + # We got a response + resp_rx = True + if self.hosts[ip].install_status: + msg = "Patch installation was successful on %s." % self.hosts[ip].hostname + msg_info += msg + "\n" + LOG.info("host-install: %s", msg) + elif self.hosts[ip].install_reject_reason: + msg = "Patch installation rejected by %s. %s" % ( + self.hosts[ip].hostname, + self.hosts[ip].install_reject_reason) + msg_error += msg + "\n" + LOG.error("Error in host-install: %s", msg) + else: + msg = "Patch installation failed on %s." % self.hosts[ip].hostname + msg_error += msg + "\n" + LOG.error("Error in host-install: %s", msg) + + self.hosts_lock.release() + break + + self.hosts_lock.release() + + time.sleep(0.5) + + if not resp_rx: + msg = "Timeout occurred while waiting response from %s." % ip + msg_error += msg + "\n" + LOG.error("Error in host-install: %s", msg) + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + def drop_host(self, host_ip, sync_nbr=True): + msg_info = "" + msg_warning = "" + msg_error = "" + + ip = host_ip + + self.hosts_lock.acquire() + # If not in hosts table, maybe a hostname was used instead + if host_ip not in self.hosts: + try: + # Because the host may be getting dropped due to deletion, + # we may be unable to do a hostname lookup. Instead, we'll + # iterate through the table here. + for host in list(self.hosts): + if host_ip == self.hosts[host].hostname: + ip = host + break + + if ip not in self.hosts: + # Translated successfully, but IP isn't in the table. + # Raise an exception to drop out to the failure handling + raise PatchError("Host IP (%s) not in table" % ip) + except Exception: + self.hosts_lock.release() + msg = "Unknown host specified: %s" % host_ip + msg_error += msg + "\n" + LOG.error("Error in drop-host: %s", msg) + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + msg = "Running drop-host for %s (%s)" % (host_ip, ip) + LOG.info(msg) + audit_log_info(msg) + + del self.hosts[ip] + for patch_id in list(self.interim_state): + if ip in self.interim_state[patch_id]: + self.interim_state[patch_id].remove(ip) + + self.hosts_lock.release() + + if sync_nbr: + sync_msg = PatchMessageDropHostReq() + sync_msg.ip = ip + self.socket_lock.acquire() + sync_msg.send(self.sock_out) + self.socket_lock.release() + + return dict(info=msg_info, warning=msg_warning, error=msg_error) + + def is_applied(self, patch_ids): + all_applied = True + + self.patch_data_lock.acquire() + + for patch_id in patch_ids: + if patch_id not in self.patch_data.metadata: + all_applied = False + break + + if self.patch_data.metadata[patch_id]["patchstate"] != constants.APPLIED: + all_applied = False + break + + self.patch_data_lock.release() + + return all_applied + + def is_available(self, patch_ids): + all_available = True + + self.patch_data_lock.acquire() + + for patch_id in patch_ids: + if patch_id not in self.patch_data.metadata: + all_available = False + break + + if self.patch_data.metadata[patch_id]["patchstate"] != \ + constants.AVAILABLE: + all_available = False + break + + self.patch_data_lock.release() + + return all_available + + def report_app_dependencies(self, patch_ids, **kwargs): + """ + Handle report of application dependencies + """ + if "app" not in kwargs: + raise PatchInvalidRequest + + appname = kwargs.get("app") + + LOG.info("Handling app dependencies report: app=%s, patch_ids=%s", + appname, ','.join(patch_ids)) + + self.patch_data_lock.acquire() + + if len(patch_ids) == 0: + if appname in self.app_dependencies: + del self.app_dependencies[appname] + else: + self.app_dependencies[appname] = patch_ids + + try: + tmpfile, tmpfname = tempfile.mkstemp( + prefix=app_dependency_basename, + dir=constants.PATCH_STORAGE_DIR) + + os.write(tmpfile, json.dumps(self.app_dependencies).encode()) + os.close(tmpfile) + + os.rename(tmpfname, app_dependency_filename) + except Exception: + LOG.exception("Failed in report_app_dependencies") + raise PatchFail("Internal failure") + finally: + self.patch_data_lock.release() + + return True + + def query_app_dependencies(self): + """ + Query application dependencies + """ + self.patch_data_lock.acquire() + + data = self.app_dependencies + + self.patch_data_lock.release() + + return dict(data) + + +# The wsgiref.simple_server module has an error handler that catches +# and prints any exceptions that occur during the API handling to stderr. +# This means the patching sys.excepthook handler that logs uncaught +# exceptions is never called, and those exceptions are lost. +# +# To get around this, we're subclassing the simple_server.ServerHandler +# in order to replace the handle_error method with a custom one that +# logs the exception instead, and will set a global flag to shutdown +# the server and reset. +# +class MyServerHandler(simple_server.ServerHandler): + def handle_error(self): + LOG.exception('An uncaught exception has occurred:') + if not self.headers_sent: + self.result = self.error_output(self.environ, self.start_response) + self.finish_response() + global keep_running + keep_running = False + + +def get_handler_cls(): + cls = simple_server.WSGIRequestHandler + + # old-style class doesn't support super + class MyHandler(cls, object): + def address_string(self): + # In the future, we could provide a config option to allow reverse DNS lookup + return self.client_address[0] + + # Overload the handle function to use our own MyServerHandler + def handle(self): + """Handle a single HTTP request""" + + self.raw_requestline = self.rfile.readline() + if not self.parse_request(): # An error code has been sent, just exit + return + + handler = MyServerHandler( + self.rfile, self.wfile, self.get_stderr(), self.get_environ() + ) + handler.request_handler = self # pylint: disable=attribute-defined-outside-init + handler.run(self.server.get_app()) + + return MyHandler + + +class PatchControllerApiThread(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + self.wsgi = None + + def run(self): + host = "127.0.0.1" + port = cfg.api_port + + try: + # In order to support IPv6, server_class.address_family must be + # set to the correct address family. Because the unauthenticated + # API always uses IPv4 for the loopback address, the address_family + # variable cannot be set directly in the WSGIServer class, so a + # local subclass needs to be created for the call to make_server, + # where the correct address_family can be specified. + class server_class(simple_server.WSGIServer): + pass + + server_class.address_family = socket.AF_INET + self.wsgi = simple_server.make_server( + host, port, + app.VersionSelectorApplication(), + server_class=server_class, + handler_class=get_handler_cls()) + + self.wsgi.socket.settimeout(api_socket_timeout) + global keep_running + while keep_running: + self.wsgi.handle_request() + + # Call garbage collect after wsgi request is handled, + # to ensure any open file handles are closed in the case + # of an upload. + gc.collect() + except Exception: + # Log all exceptions + LOG.exception("Error occurred during request processing") + + global thread_death + thread_death.set() + + def kill(self): + # Must run from other thread + if self.wsgi is not None: + self.wsgi.shutdown() + + +class PatchControllerAuthApiThread(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + # LOG.info ("Initializing Authenticated API thread") + self.wsgi = None + + def run(self): + host = CONF.auth_api_bind_ip + port = CONF.auth_api_port + if host is None: + host = utils.get_versioned_address_all() + try: + # Can only launch authenticated server post-config + while not os.path.exists('/etc/platform/.initial_config_complete'): + time.sleep(5) + + # In order to support IPv6, server_class.address_family must be + # set to the correct address family. Because the unauthenticated + # API always uses IPv4 for the loopback address, the address_family + # variable cannot be set directly in the WSGIServer class, so a + # local subclass needs to be created for the call to make_server, + # where the correct address_family can be specified. + class server_class(simple_server.WSGIServer): + pass + + server_class.address_family = utils.get_management_family() + self.wsgi = simple_server.make_server( + host, port, + auth_app.VersionSelectorApplication(), + server_class=server_class, + handler_class=get_handler_cls()) + + # self.wsgi.serve_forever() + self.wsgi.socket.settimeout(api_socket_timeout) + + global keep_running + while keep_running: + self.wsgi.handle_request() + + # Call garbage collect after wsgi request is handled, + # to ensure any open file handles are closed in the case + # of an upload. + gc.collect() + except Exception: + # Log all exceptions + LOG.exception("Authorized API failure: Error occurred during request processing") + + def kill(self): + # Must run from other thread + if self.wsgi is not None: + self.wsgi.shutdown() + + +class PatchControllerMainThread(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + # LOG.info ("Initializing Main thread") + + def run(self): + global pc + global thread_death + + # LOG.info ("In Main thread") + + try: + sock_in = pc.setup_socket() + + while sock_in is None: + # Check every thirty seconds? + # Once we've got a conf file, tied into packstack, + # we'll get restarted when the file is updated, + # and this should be unnecessary. + time.sleep(30) + sock_in = pc.setup_socket() + + # Ok, now we've got our socket. Let's start with a hello! + pc.socket_lock.acquire() + + hello = PatchMessageHello() + hello.send(pc.sock_out) + + hello_agent = PatchMessageHelloAgent() + hello_agent.send(pc.sock_out) + + pc.socket_lock.release() + + # Send hello every thirty seconds + hello_timeout = time.time() + 30.0 + remaining = 30 + + agent_query_conns = [] + + while True: + # Check to see if any other thread has died + if thread_death.is_set(): + LOG.info("Detected thread death. Terminating") + return + + # Check for in-service patch restart flag + if os.path.exists(insvc_patch_restart_controller): + LOG.info("In-service patch restart flag detected. Exiting.") + global keep_running + keep_running = False + os.remove(insvc_patch_restart_controller) + return + + inputs = [pc.sock_in] + agent_query_conns + outputs = [] + + # LOG.info("Running select, remaining=%d", remaining) + rlist, wlist, xlist = select.select(inputs, outputs, inputs, remaining) + + if (len(rlist) == 0 and + len(wlist) == 0 and + len(xlist) == 0): + # Timeout hit + pc.audit_socket() + + # LOG.info("Checking sockets") + for s in rlist: + data = '' + addr = None + msg = None + + if s == pc.sock_in: + # Receive from UDP + pc.socket_lock.acquire() + data, addr = s.recvfrom(1024) + pc.socket_lock.release() + else: + # Receive from TCP + while True: + try: + packet = s.recv(1024) + except socket.error: + LOG.exception("Socket error on recv") + data = '' + break + + if packet: + data += packet.decode() + + if data == '': + break + try: + json.loads(data) + break + except ValueError: + # Message is incomplete + continue + else: + LOG.info('End of TCP message received') + break + + if data == '': + # Connection dropped + agent_query_conns.remove(s) + s.close() + continue + + # Get the TCP endpoint address + addr = s.getpeername() + + msgdata = json.loads(data) + + # For now, discard any messages that are not msgversion==1 + if 'msgversion' in msgdata and msgdata['msgversion'] != 1: + continue + + if 'msgtype' in msgdata: + if msgdata['msgtype'] == messages.PATCHMSG_HELLO: + msg = PatchMessageHello() + elif msgdata['msgtype'] == messages.PATCHMSG_HELLO_ACK: + msg = PatchMessageHelloAck() + elif msgdata['msgtype'] == messages.PATCHMSG_SYNC_REQ: + msg = PatchMessageSyncReq() + elif msgdata['msgtype'] == messages.PATCHMSG_SYNC_COMPLETE: + msg = PatchMessageSyncComplete() + elif msgdata['msgtype'] == messages.PATCHMSG_HELLO_AGENT_ACK: + msg = PatchMessageHelloAgentAck() + elif msgdata['msgtype'] == messages.PATCHMSG_QUERY_DETAILED_RESP: + msg = PatchMessageQueryDetailedResp() + elif msgdata['msgtype'] == messages.PATCHMSG_AGENT_INSTALL_RESP: + msg = PatchMessageAgentInstallResp() + elif msgdata['msgtype'] == messages.PATCHMSG_DROP_HOST_REQ: + msg = PatchMessageDropHostReq() + + if msg is None: + msg = messages.PatchMessage() + + msg.decode(msgdata) + if s == pc.sock_in: + msg.handle(pc.sock_out, addr) + else: + msg.handle(s, addr) + + # We can drop the connection after a query response + if msg.msgtype == messages.PATCHMSG_QUERY_DETAILED_RESP and s != pc.sock_in: + agent_query_conns.remove(s) + s.shutdown(socket.SHUT_RDWR) + s.close() + + while len(stale_hosts) > 0 and len(agent_query_conns) <= 5: + ip = stale_hosts.pop() + try: + agent_sock = socket.create_connection((ip, cfg.agent_port)) + query = PatchMessageQueryDetailed() + query.send(agent_sock) + agent_query_conns.append(agent_sock) + except Exception: + # Put it back on the list + stale_hosts.append(ip) + + remaining = int(hello_timeout - time.time()) + if remaining <= 0 or remaining > 30: + hello_timeout = time.time() + 30.0 + remaining = 30 + + pc.socket_lock.acquire() + + hello = PatchMessageHello() + hello.send(pc.sock_out) + + hello_agent = PatchMessageHelloAgent() + hello_agent.send(pc.sock_out) + + pc.socket_lock.release() + + # Age out neighbours + pc.controller_neighbours_lock.acquire() + nbrs = list(pc.controller_neighbours) + for n in nbrs: + # Age out controllers after 2 minutes + if pc.controller_neighbours[n].get_age() >= 120: + LOG.info("Aging out controller %s from table", n) + del pc.controller_neighbours[n] + pc.controller_neighbours_lock.release() + + pc.hosts_lock.acquire() + nbrs = list(pc.hosts) + for n in nbrs: + # Age out hosts after 1 hour + if pc.hosts[n].get_age() >= 3600: + LOG.info("Aging out host %s from table", n) + del pc.hosts[n] + for patch_id in list(pc.interim_state): + if n in pc.interim_state[patch_id]: + pc.interim_state[patch_id].remove(n) + + pc.hosts_lock.release() + except Exception: + # Log all exceptions + LOG.exception("Error occurred during request processing") + thread_death.set() + + +def main(): + # The following call to CONF is to ensure the oslo config + # has been called to specify a valid config dir. + # Otherwise oslo_policy will fail when it looks for its files. + CONF( + (), # Required to load an anonymous configuration + default_config_files=['/etc/software/software.conf', ] + ) + + configure_logging() + + cfg.read_config() + + # daemon.pidlockfile.write_pid_to_pidfile(pidfile_path) + + global thread_death + thread_death = threading.Event() + + # Set the TMPDIR environment variable to /scratch so that any modules + # that create directories with tempfile will not use /tmp + os.environ['TMPDIR'] = '/scratch' + + global pc + pc = PatchController() + + LOG.info("launching") + api_thread = PatchControllerApiThread() + auth_api_thread = PatchControllerAuthApiThread() + main_thread = PatchControllerMainThread() + + api_thread.start() + auth_api_thread.start() + main_thread.start() + + thread_death.wait() + global keep_running + keep_running = False + + api_thread.join() + auth_api_thread.join() + main_thread.join() diff --git a/software/software/software_functions.py b/software/software/software_functions.py new file mode 100644 index 00000000..7f66e643 --- /dev/null +++ b/software/software/software_functions.py @@ -0,0 +1,987 @@ +""" +Copyright (c) 2023 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" + +import getopt +import glob +import hashlib +import logging +import os +import platform +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +from lxml import etree as ElementTree +from xml.dom import minidom + +from software.release_verify import verify_files +from software.release_verify import cert_type_all +from software.release_signing import sign_files +from software.exceptions import MetadataFail +from software.exceptions import PatchFail +from software.exceptions import PatchValidationFailure +from software.exceptions import PatchMismatchFailure + +import software.constants as constants + +try: + # The tsconfig module is only available at runtime + from tsconfig.tsconfig import SW_VERSION +except Exception: + SW_VERSION = "unknown" + +# Constants +patch_dir = constants.PATCH_STORAGE_DIR +avail_dir = "%s/metadata/available" % patch_dir +applied_dir = "%s/metadata/applied" % patch_dir +committed_dir = "%s/metadata/committed" % patch_dir +semantics_dir = "%s/semantics" % patch_dir + +# these next 4 variables may need to change to support ostree +repo_root_dir = "/var/www/pages/updates" +repo_dir = {SW_VERSION: "%s/rel-%s" % (repo_root_dir, SW_VERSION)} + +root_package_dir = "%s/packages" % patch_dir +root_scripts_dir = "/opt/software/software-scripts" +package_dir = {SW_VERSION: "%s/%s" % (root_package_dir, SW_VERSION)} + +logfile = "/var/log/software.log" +apilogfile = "/var/log/software-api.log" + +LOG = logging.getLogger('main_logger') +auditLOG = logging.getLogger('audit_logger') +audit_log_msg_prefix = 'User: sysadmin/admin Action: ' + +detached_signature_file = "signature.v2" + + +def handle_exception(exc_type, exc_value, exc_traceback): + """ + Exception handler to log any uncaught exceptions + """ + LOG.error("Uncaught exception", + exc_info=(exc_type, exc_value, exc_traceback)) + sys.__excepthook__(exc_type, exc_value, exc_traceback) + + +def configure_logging(logtofile=True, level=logging.INFO): + if logtofile: + my_exec = os.path.basename(sys.argv[0]) + + log_format = '%(asctime)s: ' \ + + my_exec + '[%(process)s]: ' \ + + '%(filename)s(%(lineno)s): ' \ + + '%(levelname)s: %(message)s' + + formatter = logging.Formatter(log_format, datefmt="%FT%T") + + LOG.setLevel(level) + main_log_handler = logging.FileHandler(logfile) + main_log_handler.setFormatter(formatter) + LOG.addHandler(main_log_handler) + + try: + os.chmod(logfile, 0o640) + except Exception: + pass + + auditLOG.setLevel(level) + api_log_handler = logging.FileHandler(apilogfile) + api_log_handler.setFormatter(formatter) + auditLOG.addHandler(api_log_handler) + try: + os.chmod(apilogfile, 0o640) + except Exception: + pass + + # Log uncaught exceptions to file + sys.excepthook = handle_exception + else: + logging.basicConfig(level=level) + + +def audit_log_info(msg=''): + msg = audit_log_msg_prefix + msg + auditLOG.info(msg) + + +def get_md5(path): + """ + Utility function for generating the md5sum of a file + :param path: Path to file + """ + md5 = hashlib.md5() + block_size = 8192 + with open(path, 'rb') as f: + for chunk in iter(lambda: f.read(block_size), b''): + md5.update(chunk) + return int(md5.hexdigest(), 16) + + +def add_text_tag_to_xml(parent, + name, + text): + """ + Utility function for adding a text tag to an XML object + :param parent: Parent element + :param name: Element name + :param text: Text value + :return:The created element + """ + tag = ElementTree.SubElement(parent, name) + tag.text = text + return tag + + +def write_xml_file(top, + fname): + # Generate the file, in a readable format if possible + outfile = open(fname, 'w') + rough_xml = ElementTree.tostring(top) + if platform.python_version() == "2.7.2": + # The 2.7.2 toprettyxml() function unnecessarily indents + # childless tags, adding whitespace. In the case of the + # yum comps.xml file, it makes the file unusable, so just + # write the rough xml + outfile.write(rough_xml) + else: + outfile.write(minidom.parseString(rough_xml).toprettyxml(indent=" ")) + + +def get_release_from_patch(patchfile): + rel = "" + try: + cmd = "tar xf %s -O metadata.tar | tar x -O" % patchfile + metadata_str = subprocess.check_output(cmd, shell=True) + root = ElementTree.fromstring(metadata_str) + # Extract release version + rel = root.findtext('sw_version') + except subprocess.CalledProcessError as e: + LOG.error("Failed to run tar command") + LOG.error("Command output: %s", e.output) + raise e + except Exception as e: + print("Failed to parse patch software version") + raise e + return rel + + +class BasePackageData(object): + """ + Information about the base package data provided by the load + """ + def __init__(self): + self.pkgs = {} + self.loaddirs() + + def loaddirs(self): + # Load up available package info + base_dir = constants.FEED_OSTREE_BASE_DIR + if not os.path.exists(base_dir): + # Return, since this could be running off-box + return + + # Look for release dirs + for reldir in glob.glob("%s/rel-*" % base_dir): + pattern = re.compile("%s/rel-(.*)" % base_dir) + m = pattern.match(reldir) + sw_rel = m.group(1) + + if sw_rel in self.pkgs: + # We've already parsed this dir once + continue + + self.pkgs[sw_rel] = {} + + # Clean up deleted data + for sw_rel in self.pkgs: + if not os.path.exists("%s/rel-%s" % (base_dir, sw_rel)): + del self.pkgs[sw_rel] + + def check_release(self, sw_rel): + return (sw_rel in self.pkgs) + + def find_version(self, sw_rel, pkgname, arch): + if sw_rel not in self.pkgs or \ + pkgname not in self.pkgs[sw_rel] or \ + arch not in self.pkgs[sw_rel][pkgname]: + return None + + return self.pkgs[sw_rel][pkgname][arch] + + +class PatchData(object): + """ + Aggregated patch data + """ + def __init__(self): + # + # The metadata dict stores all metadata associated with a patch. + # This dict is keyed on patch_id, with metadata for each patch stored + # in a nested dict. (See parse_metadata method for more info) + # + self.metadata = {} + + # + # The contents dict stores the lists of RPMs provided by each patch, + # indexed by patch_id. + # + self.contents = {} + + def add_patch(self, new_patch): + # We can just use "update" on these dicts because they are indexed by patch_id + self.metadata.update(new_patch.metadata) + self.contents.update(new_patch.contents) + + def update_patch(self, updated_patch): + for patch_id in list(updated_patch.metadata): + # Update all fields except repostate + cur_repostate = self.metadata[patch_id]['repostate'] + self.metadata[patch_id].update(updated_patch.metadata[patch_id]) + self.metadata[patch_id]['repostate'] = cur_repostate + + def delete_patch(self, patch_id): + del self.contents[patch_id] + del self.metadata[patch_id] + + @staticmethod + def modify_metadata_text(filename, + key, + value): + """ + Open an xml file, find first element matching 'key' and replace the text with 'value' + """ + new_filename = "%s.new" % filename + tree = ElementTree.parse(filename) + + # Prevent a proliferation of carriage returns when we write this XML back out to file. + for e in tree.getiterator(): + if e.text is not None: + e.text = e.text.rstrip() + if e.tail is not None: + e.tail = e.tail.rstrip() + + root = tree.getroot() + + # Make the substitution + e = root.find(key) + if e is None: + msg = "modify_metadata_text: failed to find tag '%s'" % key + LOG.error(msg) + raise PatchValidationFailure(msg) + e.text = value + + # write the modified file + outfile = open(new_filename, 'w') + rough_xml = ElementTree.tostring(root) + if platform.python_version() == "2.7.2": + # The 2.7.2 toprettyxml() function unnecessarily indents + # childless tags, adding whitespace. In the case of the + # yum comps.xml file, it makes the file unusable, so just + # write the rough xml + outfile.write(rough_xml) + else: + outfile.write(minidom.parseString(rough_xml).toprettyxml(indent=" ")) + outfile.close() + os.rename(new_filename, filename) + + def parse_metadata(self, + filename, + repostate=None): + """ + Parse an individual patch metadata XML file + :param filename: XML file + :param repostate: Indicates Applied, Available, or Committed + :return: Patch ID + """ + tree = ElementTree.parse(filename) + root = tree.getroot() + + # + # + # PATCH_0001 + # Brief description + # Longer description + # + # + # Dev + # + # + # + # + + patch_id = root.findtext("id") + if patch_id is None: + LOG.error("Patch metadata contains no id tag") + return None + + self.metadata[patch_id] = {} + + self.metadata[patch_id]["repostate"] = repostate + + # Patch state is unknown at this point + self.metadata[patch_id]["patchstate"] = "n/a" + + self.metadata[patch_id]["sw_version"] = "unknown" + + for key in ["status", + "unremovable", + "sw_version", + "summary", + "description", + "install_instructions", + "restart_script", + "warnings", + "apply_active_release_only"]: + value = root.findtext(key) + if value is not None: + self.metadata[patch_id][key] = value + + # Default reboot_required to Y + rr_value = root.findtext("reboot_required") + if rr_value is None or rr_value != "N": + self.metadata[patch_id]["reboot_required"] = "Y" + else: + self.metadata[patch_id]["reboot_required"] = "N" + + patch_sw_version = self.metadata[patch_id]["sw_version"] + global package_dir + if patch_sw_version not in package_dir: + package_dir[patch_sw_version] = "%s/%s" % (root_package_dir, patch_sw_version) + repo_dir[patch_sw_version] = "%s/rel-%s" % (repo_root_dir, patch_sw_version) + + self.metadata[patch_id]["requires"] = [] + for req in root.findall("requires"): + for req_patch in req.findall("req_patch_id"): + self.metadata[patch_id]["requires"].append(req_patch.text) + + self.contents[patch_id] = {} + + for content in root.findall("contents/ostree"): + self.contents[patch_id]["number_of_commits"] = content.findall("number_of_commits")[0].text + self.contents[patch_id]["base"] = {} + self.contents[patch_id]["base"]["commit"] = content.findall("base/commit")[0].text + self.contents[patch_id]["base"]["checksum"] = content.findall("base/checksum")[0].text + for i in range(int(self.contents[patch_id]["number_of_commits"])): + self.contents[patch_id]["commit%s" % (i + 1)] = {} + self.contents[patch_id]["commit%s" % (i + 1)]["commit"] = \ + content.findall("commit%s/commit" % (i + 1))[0].text + self.contents[patch_id]["commit%s" % (i + 1)]["checksum"] = \ + content.findall("commit%s/checksum" % (i + 1))[0].text + + return patch_id + + def load_all_metadata(self, + loaddir, + repostate=None): + """ + Parse all metadata files in the specified dir + :return: + """ + for fname in glob.glob("%s/*.xml" % loaddir): + self.parse_metadata(fname, repostate) + + def load_all(self): + # Reset the data + self.__init__() + self.load_all_metadata(applied_dir, repostate=constants.APPLIED) + self.load_all_metadata(avail_dir, repostate=constants.AVAILABLE) + self.load_all_metadata(committed_dir, repostate=constants.COMMITTED) + + def query_line(self, + patch_id, + index): + if index is None: + return None + + if index == "contents": + return self.contents[patch_id] + + if index not in self.metadata[patch_id]: + return None + + value = self.metadata[patch_id][index] + return value + + +class PatchMetadata(object): + """ + Creating metadata for a single patch + """ + def __init__(self): + self.id = None + self.sw_version = None + self.summary = None + self.description = None + self.install_instructions = None + self.warnings = None + self.status = None + self.unremovable = None + self.reboot_required = None + self.apply_active_release_only = None + self.requires = [] + self.contents = {} + + def add_rpm(self, + fname): + """ + Add an RPM to the patch + :param fname: RPM filename + :return: + """ + rpmname = os.path.basename(fname) + self.contents[rpmname] = True + + def gen_xml(self, + fname="metadata.xml"): + """ + Generate patch metadata XML file + :param fname: Path to output file + :return: + """ + top = ElementTree.Element('patch') + + add_text_tag_to_xml(top, 'id', + self.id) + add_text_tag_to_xml(top, 'sw_version', + self.sw_version) + add_text_tag_to_xml(top, 'summary', + self.summary) + add_text_tag_to_xml(top, 'description', + self.description) + add_text_tag_to_xml(top, 'install_instructions', + self.install_instructions) + add_text_tag_to_xml(top, 'warnings', + self.warnings) + add_text_tag_to_xml(top, 'status', + self.status) + add_text_tag_to_xml(top, 'unremovable', + self.unremovable) + add_text_tag_to_xml(top, 'reboot_required', + self.reboot_required) + add_text_tag_to_xml(top, 'apply_active_release_only', + self.apply_active_release_only) + + content = ElementTree.SubElement(top, 'contents') + for rpmname in sorted(list(self.contents)): + add_text_tag_to_xml(content, 'rpm', rpmname) + + req = ElementTree.SubElement(top, 'requires') + for req_patch in sorted(self.requires): + add_text_tag_to_xml(req, 'req_patch_id', req_patch) + + write_xml_file(top, fname) + + +class PatchFile(object): + """ + Patch file + """ + def __init__(self): + self.meta = PatchMetadata() + self.rpmlist = {} + + def add_rpm(self, + fname): + """ + Add an RPM to the patch + :param fname: Path to RPM + :param personality: Optional: Node type to which + the package belongs. Can be a + string or a list of strings. + :return: + """ + # Add the RPM to the metadata + self.meta.add_rpm(fname) + + # Add the RPM to the patch + self.rpmlist[os.path.abspath(fname)] = True + + def gen_patch(self, outdir): + """ + Generate the patch file, named PATCHID.patch + :param outdir: Output directory for the patch + :return: + """ + if not self.rpmlist: + raise MetadataFail("Cannot generate empty patch") + + patchfile = "%s/%s.patch" % (outdir, self.meta.id) + + # Create a temporary working directory + tmpdir = tempfile.mkdtemp(prefix="software_") + + # Save the current directory, so we can chdir back after + orig_wd = os.getcwd() + + # Change to the tmpdir + os.chdir(tmpdir) + + # Copy RPM files to tmpdir + for rpmfile in list(self.rpmlist): + shutil.copy(rpmfile, tmpdir) + + # add file signatures to RPMs + try: + subprocess.check_call(["sign-rpms", "-d", tmpdir]) + except subprocess.CalledProcessError as e: + print("Failed to to add file signatures to RPMs. Call to sign-rpms process returned non-zero exit status %i" % e.returncode) + os.chdir(orig_wd) + shutil.rmtree(tmpdir) + raise SystemExit(e.returncode) + + # generate tar file + tar = tarfile.open("software.tar", "w") + for rpmfile in list(self.rpmlist): + tar.add(os.path.basename(rpmfile)) + tar.close() + + # Generate the metadata xml file + self.meta.gen_xml("metadata.xml") + + # assemble the patch + PatchFile.write_patch(patchfile) + + # Change back to original working dir + os.chdir(orig_wd) + + shutil.rmtree(tmpdir) + + print("Patch is %s" % patchfile) + + @staticmethod + def write_patch(patchfile, cert_type=None): + # Write the patch file. Assumes we are in a directory containing + # metadata.tar and software.tar. + + # Generate the metadata tarfile + tar = tarfile.open("metadata.tar", "w") + tar.add("metadata.xml") + tar.close() + + filelist = ["metadata.tar", "software.tar"] + if os.path.exists("semantics.tar"): + filelist.append("semantics.tar") + + # Generate the signature file + sig = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF + for f in filelist: + sig ^= get_md5(f) + + sigfile = open("signature", "w") + sigfile.write("%x" % sig) + sigfile.close() + + # Generate the detached signature + # + # Note: if cert_type requests a formal signature, but the signing key + # is not found, we'll instead sign with the 'dev' key and + # need_resign_with_formal is set to True. + need_resign_with_formal = sign_files( + filelist, + detached_signature_file, + cert_type=cert_type) + + # Create the patch + tar = tarfile.open(patchfile, "w:gz") + for f in filelist: + tar.add(f) + tar.add("signature") + tar.add(detached_signature_file) + tar.close() + + if need_resign_with_formal: + try: + # Try to ensure "sign_patch_formal.sh" will be in our PATH + if 'MY_REPO' in os.environ: + os.environ['PATH'] += os.pathsep + os.environ['MY_REPO'] + "/build-tools" + if 'MY_PATCH_REPO' in os.environ: + os.environ['PATH'] += os.pathsep + os.environ['MY_PATCH_REPO'] + "/build-tools" + + # Note: This can fail if the user is not authorized to sign with the formal key. + subprocess.check_call(["sign_patch_formal.sh", patchfile]) + except subprocess.CalledProcessError as e: + print("Failed to sign official patch. Call to sign_patch_formal.sh process returned non-zero exit status %i" % e.returncode) + raise SystemExit(e.returncode) + + @staticmethod + def read_patch(path, cert_type=None): + # We want to enable signature checking by default + # Note: cert_type=None is required if we are to enforce 'no dev patches on a formal load' rule. + + # Open the patch file and extract the contents to the current dir + tar = tarfile.open(path, "r:gz") + + filelist = [] + for f in tar.getmembers(): + filelist.append(f.name) + + if detached_signature_file not in filelist: + msg = "Patch not signed" + LOG.warning(msg) + + for f in filelist: + tar.extract(f) + + # Filelist used for signature validation and verification + sig_filelist = ["metadata.tar", "software.tar"] + if "semantics.tar" in filelist: + sig_filelist.append("semantics.tar") + + # Verify the data integrity signature first + sigfile = open("signature", "r") + sig = int(sigfile.read(), 16) + sigfile.close() + + expected_sig = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF + for f in sig_filelist: + sig ^= get_md5(f) + + if sig != expected_sig: + msg = "Patch failed verification" + LOG.error(msg) + raise PatchValidationFailure(msg) + + # Verify detached signature + if os.path.exists(detached_signature_file): + sig_valid = verify_files( + sig_filelist, + detached_signature_file, + cert_type=cert_type) + if sig_valid is True: + msg = "Signature verified, patch has been signed" + if cert_type is None: + LOG.info(msg) + else: + msg = "Signature check failed" + if cert_type is None: + LOG.error(msg) + raise PatchValidationFailure(msg) + else: + msg = "Patch has not been signed" + if cert_type is None: + LOG.error(msg) + raise PatchValidationFailure(msg) + + tar = tarfile.open("metadata.tar") + tar.extractall() + + @staticmethod + def query_patch(patch, field=None): + + abs_patch = os.path.abspath(patch) + + # Create a temporary working directory + tmpdir = tempfile.mkdtemp(prefix="patch_") + + # Save the current directory, so we can chdir back after + orig_wd = os.getcwd() + + # Change to the tmpdir + os.chdir(tmpdir) + + r = {} + + try: + if field is None or field == "cert": + # Need to determine the cert_type + for cert_type_str in cert_type_all: + try: + PatchFile.read_patch(abs_patch, cert_type=[cert_type_str]) + except PatchValidationFailure: + pass + else: + # Successfully opened the file for reading, and we have discovered the cert_type + r["cert"] = cert_type_str + break + + if "cert" not in r: + # If cert is unknown, then file is not yet open for reading. + # Try to open it for reading now, using all available keys. + # We can't omit cert_type, or pass None, because that will trigger the code + # path used by installed product, in which dev keys are not accepted unless + # a magic file exists. + PatchFile.read_patch(abs_patch, cert_type=cert_type_all) + + thispatch = PatchData() + patch_id = thispatch.parse_metadata("metadata.xml") + + if field is None or field == "id": + r["id"] = patch_id + + if field is None: + for f in ["status", "sw_version", "unremovable", "summary", + "description", "install_instructions", + "warnings", "reboot_required", "apply_active_release_only"]: + r[f] = thispatch.query_line(patch_id, f) + else: + if field not in ['id', 'cert']: + r[field] = thispatch.query_line(patch_id, field) + + except PatchValidationFailure as e: + msg = "Patch validation failed during extraction" + LOG.exception(msg) + raise e + except PatchMismatchFailure as e: + msg = "Patch Mismatch during extraction" + LOG.exception(msg) + raise e + except tarfile.TarError: + msg = "Failed during patch extraction" + LOG.exception(msg) + raise PatchValidationFailure(msg) + finally: + # Change back to original working dir + os.chdir(orig_wd) + shutil.rmtree(tmpdir) + + return r + + @staticmethod + def modify_patch(patch, + key, + value): + rc = False + abs_patch = os.path.abspath(patch) + new_abs_patch = "%s.new" % abs_patch + + # Create a temporary working directory + tmpdir = tempfile.mkdtemp(prefix="patch_") + + # Save the current directory, so we can chdir back after + orig_wd = os.getcwd() + + # Change to the tmpdir + os.chdir(tmpdir) + + try: + cert_type = None + meta_data = PatchFile.query_patch(abs_patch) + if 'cert' in meta_data: + cert_type = meta_data['cert'] + PatchFile.read_patch(abs_patch, cert_type=cert_type) + PatchData.modify_metadata_text("metadata.xml", key, value) + PatchFile.write_patch(new_abs_patch, cert_type=cert_type) + os.rename(new_abs_patch, abs_patch) + rc = True + + except PatchValidationFailure as e: + raise e + except PatchMismatchFailure as e: + raise e + except tarfile.TarError: + msg = "Failed during patch extraction" + LOG.exception(msg) + raise PatchValidationFailure(msg) + except Exception as e: + template = "An exception of type {0} occurred. Arguments:\n{1!r}" + message = template.format(type(e).__name__, e.args) + print(message) + finally: + # Change back to original working dir + os.chdir(orig_wd) + shutil.rmtree(tmpdir) + + return rc + + @staticmethod + def extract_patch(patch, + metadata_dir=avail_dir, + metadata_only=False, + existing_content=None, + base_pkgdata=None): + """ + Extract the metadata and patch contents + :param patch: Patch file + :param metadata_dir: Directory to store the metadata XML file + :return: + """ + thispatch = None + + abs_patch = os.path.abspath(patch) + abs_metadata_dir = os.path.abspath(metadata_dir) + # Create a temporary working directory + tmpdir = tempfile.mkdtemp(prefix="patch_") + + # Save the current directory, so we can chdir back after + orig_wd = os.getcwd() + + # Change to the tmpdir + os.chdir(tmpdir) + + try: + # Open the patch file and extract the contents to the tmpdir + PatchFile.read_patch(abs_patch) + + thispatch = PatchData() + patch_id = thispatch.parse_metadata("metadata.xml") + + if patch_id is None: + print("Failed to import patch") + # Change back to original working dir + os.chdir(orig_wd) + shutil.rmtree(tmpdir) + return None + + if not metadata_only and base_pkgdata is not None: + # Run version validation tests first + patch_sw_version = thispatch.metadata[patch_id]["sw_version"] + if not base_pkgdata.check_release(patch_sw_version): + msg = "Patch %s software release (%s) is not installed" % (patch_id, patch_sw_version) + LOG.exception(msg) + raise PatchValidationFailure(msg) + + if metadata_only: + # This is a re-import. Ensure the content lines up + if existing_content is None \ + or existing_content != thispatch.contents[patch_id]: + msg = "Contents of re-imported patch do not match" + LOG.exception(msg) + raise PatchMismatchFailure(msg) + + patch_sw_version = thispatch.metadata[patch_id]["sw_version"] + abs_ostree_tar_dir = package_dir[patch_sw_version] + if not os.path.exists(abs_ostree_tar_dir): + os.makedirs(abs_ostree_tar_dir) + + shutil.move("metadata.xml", + "%s/%s-metadata.xml" % (abs_metadata_dir, patch_id)) + shutil.move("software.tar", + "%s/%s-software.tar" % (abs_ostree_tar_dir, patch_id)) + + # restart_script may not exist in metadata. + if thispatch.metadata[patch_id].get("restart_script"): + if not os.path.exists(root_scripts_dir): + os.makedirs(root_scripts_dir) + restart_script_name = thispatch.metadata[patch_id]["restart_script"] + shutil.move(restart_script_name, + "%s/%s" % (root_scripts_dir, restart_script_name)) + + except PatchValidationFailure as e: + raise e + except PatchMismatchFailure as e: + raise e + except tarfile.TarError: + msg = "Failed during patch extraction" + LOG.exception(msg) + raise PatchValidationFailure(msg) + except KeyError: + msg = "Failed during patch extraction" + LOG.exception(msg) + raise PatchValidationFailure(msg) + except OSError: + msg = "Failed during patch extraction" + LOG.exception(msg) + raise PatchFail(msg) + except IOError: # pylint: disable=duplicate-except + msg = "Failed during patch extraction" + LOG.exception(msg) + raise PatchFail(msg) + finally: + # Change back to original working dir + os.chdir(orig_wd) + shutil.rmtree(tmpdir) + + return thispatch + + +def patch_build(): + configure_logging(logtofile=False) + + try: + opts, remainder = getopt.getopt(sys.argv[1:], + '', + ['id=', + 'release=', + 'summary=', + 'status=', + 'unremovable', + 'reboot-required=', + 'desc=', + 'warn=', + 'inst=', + 'req=', + 'controller=', + 'controller-worker=', + 'controller-worker-lowlatency=', + 'worker=', + 'worker-lowlatency=', + 'storage=', + 'all-nodes=', + 'pre-apply=', + 'pre-remove=', + 'apply-active-release-only']) + except getopt.GetoptError: + print("Usage: %s [ ] ... " + % os.path.basename(sys.argv[0])) + print("Options:") + print("\t--id Patch ID") + print("\t--release Platform release version") + print("\t--status Patch Status Code (ie. O, R, V)") + print("\t--unremovable Marks patch as unremovable") + print("\t--reboot-required Marks patch as reboot-required (default=Y)") + print("\t--summary Patch Summary") + print("\t--desc Patch Description") + print("\t--warn Patch Warnings") + print("\t--inst Patch Install Instructions") + print("\t--req Required Patch") + print("\t--controller New package for controller") + print("\t--worker New package for worker node") + print("\t--worker-lowlatency New package for worker-lowlatency node") + print("\t--storage New package for storage node") + print("\t--controller-worker New package for combined node") + print("\t--controller-worker-lowlatency New package for lowlatency combined node") + print("\t--all-nodes New package for all node types") + print("\t--pre-apply