diff --git a/centos_iso_image.inc b/centos_iso_image.inc index 379a589409..316cdeec64 100644 --- a/centos_iso_image.inc +++ b/centos_iso_image.inc @@ -14,6 +14,9 @@ controllerconfig # storageconfig storageconfig +# cert-mon +cert-mon + # cgts-client cgts-client diff --git a/centos_pkg_dirs b/centos_pkg_dirs index 8e34aa3625..61612ffe5f 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -1,6 +1,7 @@ workerconfig controllerconfig storageconfig +sysinv/cert-mon sysinv/cgts-client sysinv/sysinv-agent sysinv/sysinv-fpga-agent diff --git a/sysinv/cert-mon/LICENSE b/sysinv/cert-mon/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/sysinv/cert-mon/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sysinv/cert-mon/PKG-INFO b/sysinv/cert-mon/PKG-INFO new file mode 100644 index 0000000000..058958020c --- /dev/null +++ b/sysinv/cert-mon/PKG-INFO @@ -0,0 +1,12 @@ +Metadata-Version: 1.1 +Name: cert-mon +Version: 1.0 +Summary: StarlingX Certificate Montior Package +Home-page: +Author: Windriver +Author-email: info@windriver.com +License: Apache-2.0 + +Description: StarlingX Certificate Monitor Package + +Platform: UNKNOWN diff --git a/sysinv/cert-mon/centos/build_srpm.data b/sysinv/cert-mon/centos/build_srpm.data new file mode 100644 index 0000000000..3701bdac03 --- /dev/null +++ b/sysinv/cert-mon/centos/build_srpm.data @@ -0,0 +1,4 @@ +SRC_DIR="." +COPY_LIST_TO_TAR="LICENSE" +EXCLUDE_LIST_FROM_TAR="centos opensuse" +TIS_PATCH_VER=PKG_GITREVCOUNT diff --git a/sysinv/cert-mon/centos/cert-mon.spec b/sysinv/cert-mon/centos/cert-mon.spec new file mode 100644 index 0000000000..58d94914e0 --- /dev/null +++ b/sysinv/cert-mon/centos/cert-mon.spec @@ -0,0 +1,45 @@ +Summary: StarlingX Certificate Monitor Package +Name: cert-mon +Version: 1.0 +Release: %{tis_patch_ver}%{?_tis_dist} +License: Apache-2.0 +Group: base +Packager: Wind River +URL: unknown +Source0: %{name}-%{version}.tar.gz + +BuildRequires: systemd-devel + +%define ocf_resourced /usr/lib/ocf/resource.d + +%description +StarlingX Certificate Monitor Package + +%define local_etc_initd /etc/init.d/ + +%define debug_package %{nil} + +%prep +%setup + +%build + +%install +install -m 755 -p -D cert-mon %{buildroot}/usr/lib/ocf/resource.d/platform/cert-mon +install -m 644 -p -D cert-mon.service %{buildroot}%{_unitdir}/cert-mon.service + +%post + + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%doc LICENSE + +# SM OCF Start/Stop/Monitor Scripts +%{ocf_resourced}/platform/cert-mon + +# systemctl service files +%{_unitdir}/cert-mon.service diff --git a/sysinv/cert-mon/cert-mon b/sysinv/cert-mon/cert-mon new file mode 100644 index 0000000000..c877be1242 --- /dev/null +++ b/sysinv/cert-mon/cert-mon @@ -0,0 +1,353 @@ +#!/bin/sh +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# +# Support: www.windriver.com +# +####################################################################### +# Initialization: + +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +binname="cert-mon" + +####################################################################### + +# Fill in some defaults if no values are specified +OCF_RESKEY_binary_default=${binname} +OCF_RESKEY_dbg_default="false" +OCF_RESKEY_user_default="root" +OCF_RESKEY_pid_default="/var/run/${binname}.pid" +OCF_RESKEY_config_default="/etc/sysinv/cert-mon.conf" + +: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}} +: ${OCF_RESKEY_dbg=${OCF_RESKEY_dbg_default}} +: ${OCF_RESKEY_user=${OCF_RESKEY_user_default}} +: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}} +: ${OCF_RESKEY_config=${OCF_RESKEY_config_default}} +: ${OCF_RESKEY_client_binary=${OCF_RESKEY_client_binary_default}} + +mydaemon="/usr/bin/${OCF_RESKEY_binary}" + +####################################################################### + +usage() { + cat < + + +1.0 + + +This 'cert-mon' is an OCF Compliant Resource Agent that manages start, stop +and in-service monitoring of the Certificate Monitor Process + + + +Manages the Certificate Monitor (cert-mon) process + + + + + + + +dbg = false ... info, warn and err logs sent to output stream (default) +dbg = true ... Additional debug logs are also sent to the output stream + +Service Debug Control Option + + + + + +User running Certificate Monitor Service (cert-mon) + +Certificate Monitor Service (cert-mon) user + + + + + + + + + + + + + + +END + return ${OCF_SUCCESS} +} + +cert_mon_validate() { + + local rc + + proc="${binname}:validate" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + check_binary ${OCF_RESKEY_binary} + + if [ ! -f ${OCF_RESKEY_config} ] ; then + ocf_log err "${OCF_RESKEY_binary} ini file missing (${OCF_RESKEY_config})" + return ${OCF_ERR_CONFIGURED} + fi + + getent passwd $OCF_RESKEY_user >/dev/null 2>&1 + rc=$? + if [ $rc -ne 0 ]; then + ocf_log err "User $OCF_RESKEY_user doesn't exist" + return ${OCF_ERR_CONFIGURED} + fi + + return ${OCF_SUCCESS} +} + +cert_mon_status() { + local pid + local rc + + proc="${binname}:status" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + if [ ! -f $OCF_RESKEY_pid ]; then + ocf_log info "${binname}:Certificate Monitor (cert-mon) is not running" + return $OCF_NOT_RUNNING + else + pid=`cat $OCF_RESKEY_pid` + fi + + ocf_run -warn kill -s 0 $pid + rc=$? + if [ $rc -eq 0 ]; then + return $OCF_SUCCESS + else + ocf_log info "${binname}:Old PID file found, but Certificate Monitor Service (cert-mon) is not running" + rm -f $OCF_RESKEY_pid + return $OCF_NOT_RUNNING + fi +} + +cert_mon_monitor () { + local rc + + cert_mon_status + rc=$? + # If status returned anything but success, return that immediately + if [ $rc -ne $OCF_SUCCESS ]; then + return $rc + fi + + ocf_log debug "Certificate Monitor Service (cert-mon) monitor succeeded" + + return $OCF_SUCCESS +} + +cert_mon_start () { + local rc + + cert_mon_status + rc=$? + if [ $rc -ne ${OCF_SUCCESS} ] ; then + ocf_log err "${proc} ping test failed (rc=${rc})" + cert_mon_stop + else + return ${OCF_SUCCESS} + fi + + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + RUN_OPT_DEBUG="--debug" + else + RUN_OPT_DEBUG="" + fi + + su ${OCF_RESKEY_user} -s /bin/sh -c "${OCF_RESKEY_binary} --config-file=${OCF_RESKEY_config} ${RUN_OPT_DEBUG}"' >> /dev/null 2>&1 & echo $!' > $OCF_RESKEY_pid + rc=$? + if [ ${rc} -ne ${OCF_SUCCESS} ] ; then + ocf_log err "${proc} failed ${mydaemon} daemon (rc=$rc)" + return ${OCF_ERR_GENERIC} + else + if [ -f ${OCF_RESKEY_pid} ] ; then + pid=`cat ${OCF_RESKEY_pid}` + ocf_log info "${proc} running with pid ${pid}" + else + ocf_log info "${proc} with no pid file" + fi + fi + + # Record success or failure and return status + if [ ${rc} -eq $OCF_SUCCESS ] ; then + ocf_log info "Certificate Monitor Service (${OCF_RESKEY_binary}) started (pid=${pid})" + else + ocf_log err "Certificate Monitor (${OCF_RESKEY_binary}) failed to start (rc=${rc})" + rc=${OCF_NOT_RUNNING} + fi + + return ${rc} +} + +cert_mon_confirm_stop() { + local my_bin + local my_processes + + my_binary=`which ${OCF_RESKEY_binary}` + my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2|/usr/bin/python3) ${my_binary}([^\w-]|$)"` + + if [ -n "${my_processes}" ] + then + ocf_log info "About to SIGKILL the following: ${my_processes}" + pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2|/usr/bin/python3) ${my_binary}([^\w-]|$)" + fi +} + +cert_mon_stop () { + local rc + local pid + + cert_mon_status + rc=$? + if [ $rc -eq $OCF_NOT_RUNNING ]; then + ocf_log info "${proc} Certificate Monitor (cert-mon) already stopped" + cert_mon_confirm_stop + return ${OCF_SUCCESS} + fi + + # Try SIGTERM + pid=`cat $OCF_RESKEY_pid` + ocf_run kill -s TERM $pid + rc=$? + if [ $rc -ne 0 ]; then + ocf_log err "${proc} Certificate Monitor (cert-mon) couldn't be stopped" + cert_mon_confirm_stop + exit $OCF_ERR_GENERIC + fi + + # stop waiting + shutdown_timeout=15 + if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then + shutdown_timeout=$((($OCF_RESKEY_CRM_meta_timeout/1000)-5)) + fi + count=0 + while [ $count -lt $shutdown_timeout ]; do + cert_mon_status + rc=$? + if [ $rc -eq $OCF_NOT_RUNNING ]; then + break + fi + count=`expr $count + 1` + sleep 1 + ocf_log info "${proc} Certificate Monitor (cert-mon) still hasn't stopped yet. Waiting ..." + done + + cert_mon_status + rc=$? + if [ $rc -ne $OCF_NOT_RUNNING ]; then + # SIGTERM didn't help either, try SIGKILL + ocf_log info "${proc} Certificate Monitor (cert-mon) failed to stop after ${shutdown_timeout}s using SIGTERM. Trying SIGKILL ..." + ocf_run kill -s KILL $pid + fi + cert_mon_confirm_stop + + ocf_log info "${proc} Certificate Monitor (cert-mon) stopped." + + rm -f $OCF_RESKEY_pid + + return $OCF_SUCCESS + +} + +cert_mon_reload () { + local rc + + proc="${binname}:reload" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + cert_mon_stop + rc=$? + if [ $rc -eq ${OCF_SUCCESS} ] ; then + #sleep 1 + cert_mon_start + rc=$? + if [ $rc -eq ${OCF_SUCCESS} ] ; then + ocf_log info "Certificate Monitor (${OCF_RESKEY_binary}) process restarted" + fi + fi + + if [ ${rc} -ne ${OCF_SUCCESS} ] ; then + ocf_log err "Certificate Monitor (${OCF_RESKEY_binary}) process failed to restart (rc=${rc})" + fi + + return ${rc} +} + +case ${__OCF_ACTION} in + meta-data) meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) usage + exit ${OCF_SUCCESS} + ;; +esac + +# Anything except meta-data and help must pass validation +cert_mon_validate || exit $? + +if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${binname}:${__OCF_ACTION} action" +fi + +case ${__OCF_ACTION} in + + start) cert_mon_start + ;; + stop) cert_mon_stop + ;; + status) cert_mon_status + ;; + reload) cert_mon_reload + ;; + monitor) cert_mon_monitor + ;; + validate-all) cert_mon_validate + ;; + *) usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac diff --git a/sysinv/cert-mon/cert-mon.service b/sysinv/cert-mon/cert-mon.service new file mode 100644 index 0000000000..21a505bed0 --- /dev/null +++ b/sysinv/cert-mon/cert-mon.service @@ -0,0 +1,15 @@ +[Unit] +Description=Certificate Monitor +After=network-online.target syslog-ng.service config.service sysinv-api.service + +[Service] +Type=simple +RemainAfterExit=yes +User=root +Environment=OCF_ROOT=/usr/lib/ocf +ExecStart=/usr/lib/ocf/resource.d/platform/cert-mon start +ExecStop=/usr/lib/ocf/resource.d/platform/cert-mon stop +PIDFile=/var/run/cert-mon.pid + +[Install] +WantedBy=multi-user.target diff --git a/sysinv/sysinv/centos/sysinv.spec b/sysinv/sysinv/centos/sysinv.spec index 794a432971..a38644a25c 100644 --- a/sysinv/sysinv/centos/sysinv.spec +++ b/sysinv/sysinv/centos/sysinv.spec @@ -156,6 +156,7 @@ rm -rf $RPM_BUILD_ROOT %{_bindir}/sysinv-puppet %{_bindir}/sysinv-helm %{_bindir}/sysinv-utils +%{_bindir}/cert-mon %package wheels Summary: %{name} wheels diff --git a/sysinv/sysinv/sysinv/setup.cfg b/sysinv/sysinv/sysinv/setup.cfg index f199c2e2df..ea1f6c4b6e 100644 --- a/sysinv/sysinv/sysinv/setup.cfg +++ b/sysinv/sysinv/sysinv/setup.cfg @@ -38,6 +38,7 @@ console_scripts = sysinv-puppet = sysinv.cmd.puppet:main sysinv-helm = sysinv.cmd.helm:main sysinv-utils = sysinv.cmd.utils:main + cert-mon = sysinv.cmd.cert_mon:main systemconfig.puppet_plugins = 001_platform = sysinv.puppet.platform:PlatformPuppet @@ -64,6 +65,7 @@ systemconfig.puppet_plugins = 035_dockerdistribution = sysinv.puppet.dockerdistribution:DockerDistributionPuppet 036_pciirqaffinity = sysinv.puppet.pci_irq_affinity:PciIrqAffinityPuppet 037_monitor = sysinv.puppet.monitor:MonitorPuppet + 038_certmon = sysinv.puppet.certmon:CertMonPuppet 099_service_parameter = sysinv.puppet.service_parameter:ServiceParamPuppet systemconfig.armada.manifest_ops = diff --git a/sysinv/sysinv/sysinv/sysinv/cert_mon/__init__.py b/sysinv/sysinv/sysinv/sysinv/cert_mon/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sysinv/sysinv/sysinv/sysinv/cert_mon/certificate_mon_manager.py b/sysinv/sysinv/sysinv/sysinv/cert_mon/certificate_mon_manager.py new file mode 100644 index 0000000000..4ed7a33dcf --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/cert_mon/certificate_mon_manager.py @@ -0,0 +1,144 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# +from oslo_config import cfg +import greenlet +from eventlet import greenthread +from oslo_log import log +from oslo_service import periodic_task +import time + +from sysinv.openstack.common import context +from sysinv.cert_mon import watcher + +LOG = log.getLogger(__name__) + +cert_mon_opts = [ + cfg.IntOpt('audit_interval', + default=86400, # 24 hours + help='Interval to run certificate audit'), + cfg.IntOpt('retry_interval', + default=10, + help='interval to reattempt accessing external system ' + 'if failure occurred'), + cfg.IntOpt('max_retry', + default=5, + help='interval to reattempt accessing external system ' + 'if failure occurred'), +] + +CONF = cfg.CONF +CONF.register_opts(cert_mon_opts, 'certmon') + + +class CertificateMonManager(periodic_task.PeriodicTasks): + def __init__(self): + super(CertificateMonManager, self).__init__(CONF) + self.mon_thread = None + self.audit_thread = None + self.monitor = None + self.reattempt_tasks = [] + + def periodic_tasks(self, context, raise_on_error=False): + """Tasks to be run at a periodic interval.""" + return self.run_periodic_tasks(context, raise_on_error=raise_on_error) + + @periodic_task.periodic_task(spacing=CONF.certmon.audit_interval) + def audit_cert_task(self, context): + # [Place holder for] auditing subcloud certificate + # this task runs every very long period of time, such as 24 hours + LOG.info('Audit certificate') + + @periodic_task.periodic_task(spacing=CONF.certmon.retry_interval) + def retry_task(self, context): + # Failed tasks that need to be reattempted will be taken care here + max_attempts = CONF.certmon.max_retry + tasks = self.reattempt_tasks[:] + for task in tasks: + if task.run(): + self.reattempt_tasks.remove(task) + LOG.info('Reattempt has succeeded') + elif task.number_of_reattempt == max_attempts: + LOG.error('Maximum attempts (%s) has been reached. Give up' % + max_attempts) + if task in self.reattempt_tasks: + self.reattempt_tasks.remove(task) + + def start_audit(self): + LOG.info('Auditing interval %s' % CONF.certmon.audit_interval) + self.audit_thread = greenthread.spawn(self.audit_cert) + + def init_monitor(self): + self.monitor = watcher.CertWatcher() + self.monitor.initialize() + + def start_monitor(self): + while True: + try: + self.init_monitor() + except Exception as e: + LOG.error(e) + time.sleep(CONF.certmon.retry_interval) + else: + break + self.mon_thread = greenthread.spawn(self.monitor_cert) + + def stop_monitor(self): + if self.mon_thread: + self.mon_thread.kill() + self.mon_thread.wait() + + def stop_audit(self): + if self.audit_thread: + self.audit_thread.kill() + self.audit_thread.wait() + + def audit_cert(self): + admin_context = context.RequestContext('admin', 'admin', is_admin=True) + while True: + try: + self.run_periodic_tasks(context=admin_context) + time.sleep(1) + except greenlet.GreenletExit: + break + except Exception as e: + LOG.error(e) + + def monitor_cert(self): + while True: + # never exit until exit signal received + try: + self.monitor.start_watch( + func=lambda task: self._add_reattempt_task(task)) + except greenlet.GreenletExit: + break + except Exception as e: + # A bug somewhere? + # It shouldn't fall to here, but log and restart if it did + LOG.error(e) + + def _add_reattempt_task(self, task): + id = task.get_id() + for t in self.reattempt_tasks: + if t.get_id() == id: + self.reattempt_tasks.remove(t) + LOG.info('Older task %s is replaced with new task' % id) + break + + self.reattempt_tasks.append(task) diff --git a/sysinv/sysinv/sysinv/sysinv/cert_mon/service.py b/sysinv/sysinv/sysinv/sysinv/cert_mon/service.py new file mode 100644 index 0000000000..a064c05a25 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/cert_mon/service.py @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# +from oslo_config import cfg +from oslo_log import log as logging +from oslo_service import service + +from sysinv.cert_mon.certificate_mon_manager import CertificateMonManager + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +class CertificateMonitorService(service.Service): + """Lifecycle manager for a running audit service.""" + + def __init__(self): + super(CertificateMonitorService, self).__init__() + self.manager = CertificateMonManager() + + def start(self): + super(CertificateMonitorService, self).start() + self.manager.start_audit() + self.manager.start_monitor() + + def stop(self): + self.manager.stop_audit() + self.manager.stop_monitor() + super(CertificateMonitorService, self).stop() diff --git a/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py b/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py new file mode 100644 index 0000000000..a5d20ba411 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py @@ -0,0 +1,245 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# +import json +from oslo_config import cfg +from oslo_log import log +from six.moves.urllib.request import Request +from six.moves.urllib.error import HTTPError +from six.moves.urllib.error import URLError +from six.moves.urllib.request import urlopen + +from sysinv.common import constants +from sysinv.openstack.common.keystone_objects import Token + +LOG = log.getLogger(__name__) +CONF = cfg.CONF + + +def update_admin_ep_cert(token, ca_crt, tls_crt, tls_key): + service_type = constants.SERVICE_TYPE_PLATFORM + service_name = 'sysinv' + sysinv_url = token.get_service_internal_url(service_type, service_name) + api_cmd = sysinv_url + '/certificate/certificate_renew' + api_cmd_payload = dict() + api_cmd_payload['certtype'] = 'admin-endpoint-cert' + resp = rest_api_request(token, "POST", api_cmd, json.dumps(api_cmd_payload)) + + if 'result' in resp and resp['result'] == 'OK': + LOG.info('Request succeed') + else: + LOG.error('Request response %s' % resp) + + +def rest_api_request(token, method, api_cmd, + api_cmd_payload=None, timeout=10): + """ + Make a rest-api request + Returns: response as a dictionary + """ + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "cert-mon/1.0" + + LOG.debug("%s cmd:%s hdr:%s payload:%s" % (method, + api_cmd, api_cmd_headers, api_cmd_payload)) + + try: + request_info = Request(api_cmd) + request_info.get_method = lambda: method + if token: + request_info.add_header("X-Auth-Token", token.get_id()) + request_info.add_header("Accept", "application/json") + + if api_cmd_headers is not None: + for header_type, header_value in api_cmd_headers.items(): + request_info.add_header(header_type, header_value) + + if api_cmd_payload is not None: + request_info.add_data(api_cmd_payload) + + request = None + try: + request = urlopen(request_info, timeout=timeout) + response = request.read() + finally: + if request: + request.close() + + if response == "": + response = json.loads("{}") + else: + response = json.loads(response) + + except HTTPError as e: + if 401 == e.code: + if token: + token.set_expired() + raise + + except URLError: + LOG.error("Cannot access %s" % api_cmd) + raise + + return response + + +def get_system(token, method, api_cmd, api_cmd_headers=None, + api_cmd_payload=None, timeout=10): + """ + Make a rest-api request + Returns: response as a dictionary + """ + LOG.debug("%s cmd:%s hdr:%s payload:%s" % (method, + api_cmd, api_cmd_headers, api_cmd_payload)) + + response = None + try: + request_info = Request(api_cmd) + request_info.get_method = lambda: method + if token: + request_info.add_header("X-Auth-Token", token.get_id()) + request_info.add_header("Accept", "application/json") + + if api_cmd_headers is not None: + for header_type, header_value in api_cmd_headers.items(): + request_info.add_header(header_type, header_value) + + if api_cmd_payload is not None: + request_info.add_data(api_cmd_payload) + + request = urlopen(request_info, timeout=timeout) + response = request.read() + + if response == "": + response = json.loads("{}") + else: + response = json.loads(response) + request.close() + + except HTTPError as e: + if 401 == e.code: + if token: + token.set_expired() + LOG.warn("HTTP Error e.code=%s e=%s" % (e.code, e)) + if hasattr(e, 'msg') and e.msg: + response = json.loads(e.msg) + else: + response = json.loads("{}") + raise + + except URLError: + LOG.error("Cannot access %s" % api_cmd) + raise + + finally: + return response + + +def get_token(): + token = _get_token( + CONF.keystone_authtoken.auth_url, + CONF.keystone_authtoken.project_name, + CONF.keystone_authtoken.username, + CONF.keystone_authtoken.password, + CONF.keystone_authtoken.user_domain_name, + CONF.keystone_authtoken.project_domain_name, + CONF.keystone_authtoken.region_name) + + return token + + +def _get_token(auth_url, auth_project, username, password, user_domain, + project_domain, region_name): + """ + Ask OpenStack Keystone for a token + Returns: token object or None on failure + """ + try: + url = auth_url + "/v3/auth/tokens" + request_info = Request(url) + request_info.add_header("Content-type", "application/json") + request_info.add_header("Accept", "application/json") + payload = json.dumps( + {"auth": { + "identity": { + "methods": [ + "password" + ], + "password": { + "user": { + "name": username, + "password": password, + "domain": {"name": user_domain} + } + } + }, + "scope": { + "project": { + "name": auth_project, + "domain": {"name": project_domain} + }}}}) + + request_info.add_data(payload) + + request = urlopen(request_info) + # Identity API v3 returns token id in X-Subject-Token + # response header. + token_id = request.info().getheader('X-Subject-Token') + response = json.loads(request.read()) + request.close() + # save the region name for service url lookup + return Token(response, token_id, region_name) + + except HTTPError as e: + LOG.error("%s, %s" % (e.code, e.read())) + return None + + except URLError as e: + LOG.error(e) + return None + + +def init_keystone_auth_opts(): + keystone_opts = [ + cfg.StrOpt('username', + help='Username of account'), + cfg.StrOpt('auth_uri', + help='authentication uri'), + cfg.StrOpt('password', + help='Password of account'), + cfg.StrOpt('project_name', + help='Tenant name of account'), + cfg.StrOpt('user_domain_name', + default='Default', + help='User domain name of account'), + cfg.StrOpt('project_domain_name', + default='Default', + help='Project domain name of account'), + cfg.StrOpt('region_name', + default='', + help='Region name'), + cfg.StrOpt('auth_url', + default='', + help='authorization url'), + ] + + keystone_opt_group = cfg.OptGroup(name='keystone_authtoken', + title='Keystone options') + cfg.CONF.register_opts(keystone_opts, group=keystone_opt_group.name) diff --git a/sysinv/sysinv/sysinv/sysinv/cert_mon/watcher.py b/sysinv/sysinv/sysinv/sysinv/cert_mon/watcher.py new file mode 100644 index 0000000000..a05caeddb8 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/cert_mon/watcher.py @@ -0,0 +1,258 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# +from datetime import datetime +from dateutil.parser import parse +from kubernetes import client +from kubernetes import watch +from kubernetes.client import Configuration +from kubernetes import config +from oslo_config import cfg +from oslo_log import log + +from sysinv.cert_mon import utils +from sysinv.common import constants +from sysinv.common import kubernetes as sys_kube + +KUBE_CONFIG_PATH = '/etc/kubernetes/admin.conf' +LOG = log.getLogger(__name__) + +CERT_NAMESPACE_SYS_CONTROLLER = 'dc-cert' +CERT_NAMESPACE_SUBCLOUD_CONTROLLER = 'sc-cert' + +SECRET_ACTION_TYPE_ADDED = 'ADDED' +SECRET_ACTION_TYPE_DELETED = 'DELETED' +SECRET_ACTION_TYPE_MODIFIED = 'MODIFIED' +CONF = cfg.CONF + + +class MonitorContext(object): + def __init__(self): + self.system = None + self.dc_role = None + self._token = None + self.kubernete_namespace = None + + def initialize(self): + utils.init_keystone_auth_opts() + token = self.get_token() + service_type = 'platform' + service_name = 'sysinv' + sysinv_url = token.get_service_internal_url(service_type, + service_name) + api_cmd = sysinv_url + '/isystems' + res = utils.rest_api_request(token, "GET", api_cmd)['isystems'] + if len(res) == 1: + self.system = res[0] + self.dc_role = self.system['distributed_cloud_role'] + LOG.info('Result %s' % self.system) + else: + raise Exception('Failed to access system data') + + def get_token(self): + if not self._token or self._token.is_expired(): + self._token = utils.get_token() + return self._token + + +class CertUpdateEventData(object): + def __init__(self, event_data): + raw_obj = event_data['raw_object'] + metadata = raw_obj['metadata'] + data = raw_obj['data'] + managed_fields = metadata['managedFields'] + self.action = event_data['type'] + self.cert_name = metadata['name'] + + self.last_operation = '' + self.last_operation_time = None + if len(managed_fields) > 0: + managed_field = managed_fields[0] + self.last_operation = managed_field['operation'] + self.last_operation_time = parse(managed_field['time']).replace(tzinfo=None) + + creation_timestamp = metadata['creationTimestamp'] + self.creation_time = parse(creation_timestamp).replace(tzinfo=None) + self.ca_crt = data['ca.crt'] if 'ca.crt' in data else '' + self.tls_crt = data['tls.crt'] if 'tls.crt' in data else '' + self.tls_key = data['tls.key'] if 'tls.key' in data else '' + + def equal(self, obj): + return self.action == obj.action and \ + self.cert_name == obj.cert_name and \ + self.ca_crt == obj.ca_crt and \ + self.tls_crt == obj.tls_crt and \ + self.creation_time == obj.creation_time and \ + self.last_operation == obj.last_operation and \ + self.last_operation_time == obj.last_operation_time + + def __str__(self): + format = 'action %s (%s)\nhash: ca_crt: %s tls_crt %s tls_key %s\n' \ + 'created at %s last operation %s last update at %s' + return format % \ + ( + self.action, self.cert_name, self.hash(self.ca_crt), + self.hash(self.tls_crt), self.hash(self.tls_key), + self.creation_time, self.last_operation, + self.last_operation_time) + + @staticmethod + def hash(data): + import hashlib + m = hashlib.md5() + m.update(data) + return m.hexdigest() + + +class CertUpdateEvent(object): + def __init__(self, listener, event_data): + self.listener = listener + self.event_data = event_data + self.number_of_reattempt = 0 + + def run(self): + try: + self.listener.notify_changed(self.event_data) + except Exception as e: + LOG.error('%s Reattempt %s %s failed. %s' % + (self.number_of_reattempt, self.event_data.action, + self.event_data.cert_name, e)) + self.number_of_reattempt = self.number_of_reattempt + 1 + return False + else: + return True + + def get_id(self): + """ + Return the key id for the task. + A task will be replaced with a newer task with the same id + when it is in a queue (for reattempting) + """ + return 'cert-update: %s' % self.event_data.cert_name + + +class CertWatcherListener(object): + def __init__(self, context): + self.context = context + + def notify_changed(self, event_data): + if self.check_filter(event_data): + self.do_action(event_data) + + def check_filter(self, event_data): + return False + + def do_action(self, event_data): + pass + + +class CertWatcher(object): + def __init__(self): + self.listeners = [] + self.namespace = None + self.context = MonitorContext() + + def register_listener(self, listener): + return self.listeners.append(listener) + + def start_watch(self, func): + config.load_kube_config(KUBE_CONFIG_PATH) + c = Configuration() + c.verify_ssl = True + Configuration.set_default(c) + ccApi = client.CoreV1Api() + w = watch.Watch() + + LOG.debug('Monitor secrets in %s' % self.namespace) + for item in w.stream(ccApi.list_namespaced_secret, namespace=self.namespace): + event_data = CertUpdateEventData(item) + for listener in self.listeners: + try: + listener.notify_changed(event_data) + except Exception as e: + LOG.error(e) + reattempt = CertUpdateEvent(event_data, listener) + func(reattempt) + + def initialize(self): + self.context.initialize() + role = self.context.dc_role + LOG.info('dc role: %s' % role) + if role == constants.DISTRIBUTED_CLOUD_ROLE_SUBCLOUD: + ns = CERT_NAMESPACE_SUBCLOUD_CONTROLLER + elif role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER: + ns = CERT_NAMESPACE_SYS_CONTROLLER + else: + ns = '' + self.namespace = ns + self.context.kubernete_namespace = ns + self.register_listener(AdminEndpointRenew(self.context)) + + +class AdminEndpointRenew(CertWatcherListener): + def __init__(self, context): + super(AdminEndpointRenew, self).__init__(context) + role = self.context.dc_role + if role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER: + self.cert_name = "dc-adminep-certificate" + elif role == constants.DISTRIBUTED_CLOUD_ROLE_SUBCLOUD: + self.cert_name = "sc-adminep-certificate" + else: + self.cert_name = None + + self.monitor_start = datetime.now() + + def check_filter(self, event_data): + if self.cert_name != event_data.cert_name: + return False + + if event_data.action in (SECRET_ACTION_TYPE_ADDED, SECRET_ACTION_TYPE_MODIFIED)\ + and event_data.ca_crt and event_data.tls_crt and \ + event_data.tls_key: + return True + else: + return False + + def do_action(self, event_data): + LOG.info('%s' % event_data) + # here is a workaround for replacing private key when renewing certficate. + # when secret is deleted, the cert-manager will recreate the secret with + # new private key. + # a normal renewing scenario is + # secret updated -> delete secret -> secret added -> secret updated + # the first secret updated is triggered by the cert-manager renewing the cert + # the operation does not include rekey. Then the secret is deleted so that a new + # certificate is created with new key. Giving the fact that cert-manager + # creates new certificate secret reasonably quickly, we assume the secret update + # on a recently created secret has new key. (normal scenario certificate renew + # interval is far longer than 1 minute (in days or at least hours). + # In the very rare event if cert-manager creates new secret really slowly for a + # short period of time (takes more than 1 minutes to update the secret) + # the secret will be recreated again. This is going to be recovered when + # cert-manager normal behave is restored. + reasonable_dalay = 60 # assuming recreating secret takes less then 60 seconds + delta = (event_data.last_operation_time - event_data.creation_time).total_seconds() + if event_data.action == SECRET_ACTION_TYPE_MODIFIED and delta > reasonable_dalay: + kube_op = sys_kube.KubeOperator() + kube_op.kube_delete_secret(event_data.cert_name, self.context.kubernete_namespace) + LOG.info('Delete secret %s:%s' % (self.context.kubernete_namespace, event_data.cert_name)) + else: + token = self.context.get_token() + utils.update_admin_ep_cert(token, event_data.ca_crt, event_data.tls_crt, + event_data.tls_key) diff --git a/sysinv/sysinv/sysinv/sysinv/cmd/cert_mon.py b/sysinv/sysinv/sysinv/sysinv/cmd/cert_mon.py new file mode 100644 index 0000000000..7a2a68f460 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/cmd/cert_mon.py @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# +from oslo_config import cfg +from oslo_log import log as logging +from oslo_service import service + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +def main(): + logging.register_options(CONF) + CONF(project='sysinv', prog='certmon') + logging.set_defaults() + logging.setup(cfg.CONF, 'certmon') + + from sysinv.cert_mon import service as cert_mon + LOG.info("Configuration:") + cfg.CONF.log_opt_values(LOG, logging.INFO) + + srv = cert_mon.CertificateMonitorService() + launcher = service.launch(cfg.CONF, srv) + + launcher.wait() + + +if __name__ == '__main__': + main() diff --git a/sysinv/sysinv/sysinv/sysinv/common/utils.py b/sysinv/sysinv/sysinv/sysinv/common/utils.py index 42a0fdb5d9..2ab2600e91 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/common/utils.py @@ -2246,7 +2246,7 @@ def get_admin_ep_cert(dc_role): raise Exception for kubernetes data errors """ if dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER: - endpoint_cert_secret_name = 'dc-adminep-root-ca-certificate' + endpoint_cert_secret_name = 'dc-adminep-certificate' endpoint_cert_secret_ns = 'dc-cert' elif dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SUBCLOUD: endpoint_cert_secret_name = 'sc-adminep-certificate' @@ -2265,19 +2265,15 @@ def get_admin_ep_cert(dc_role): )) data = secret.data - if (dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER - and 'ca.crt' not in data) or \ - 'tls.crt' not in data or 'tls.key' not in data: + if 'tls.crt' not in data or 'tls.key' not in data: raise Exception("Invalid admin endpoint certificate data.") - ca_crt = None try: - if dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER: - ca_crt = base64.b64decode(data['ca.crt']) tls_crt = base64.b64decode(data['tls.crt']) tls_key = base64.b64decode(data['tls.key']) except TypeError: - raise Exception('admin endpoint root ca certification is invalid') + raise Exception('admin endpoint secret is invalid %s' % + endpoint_cert_secret_name) if dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SUBCLOUD: try: @@ -2288,9 +2284,23 @@ def get_admin_ep_cert(dc_role): # when intermediate or root ca is renewed. # but the operation should not stop here, b/c if admin endpoint # certificate is not updated, system controller may lost - # access to the subcloud admin endpoints which will make the situation - # impossible to recover. + # access to the subcloud admin endpoints which will make the + # situation impossible to recover. LOG.error('Cannot read DC root CA certificate %s' % e) + elif dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER: + root_ca_secret_name = 'dc-adminep-root-ca-certificate' + secret = kube.kube_get_secret( + root_ca_secret_name, endpoint_cert_secret_ns) + + if not hasattr(secret, 'data'): + raise Exception('Invalid secret %s\\%s' % ( + endpoint_cert_secret_ns, endpoint_cert_secret_name + )) + try: + ca_crt = base64.b64decode(data['ca.crt']) + except TypeError: + raise Exception('admin endpoint secret is invalid %s' % + root_ca_secret_name) secret_data['dc_root_ca_crt'] = ca_crt secret_data['admin_ep_crt'] = "%s%s" % (tls_key, tls_crt) diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index dec03be6bb..15f45cabcb 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -10838,7 +10838,7 @@ class ConductorManager(service.PeriodicService): """ Update admin endpoint certificate :param context: an admin context. - :return: + :return: true if certificate is renewed """ update_required = False system = self.dbapi.isystem_get_one() @@ -10846,7 +10846,7 @@ class ConductorManager(service.PeriodicService): cert_data = cutils.get_admin_ep_cert(system_dc_role) if cert_data is None: - return + return False ca_crt = cert_data['dc_root_ca_crt'] admin_ep_cert = cert_data['admin_ep_crt'] @@ -10858,17 +10858,19 @@ class ConductorManager(service.PeriodicService): else: update_required = True - if os.path.isfile(constants.DC_ROOT_CA_CERT_PATH): - with open(constants.DC_ROOT_CA_CERT_PATH, mode='r') as f: - dc_root_ca_cert = f.read() - if ca_crt not in dc_root_ca_cert: + if ca_crt is not None: + if os.path.isfile(constants.DC_ROOT_CA_CERT_PATH): + with open(constants.DC_ROOT_CA_CERT_PATH, mode='r') as f: + dc_root_ca_cert = f.read() + if ca_crt not in dc_root_ca_cert: + update_required = True + else: update_required = True - else: - update_required = True if update_required: m = hashlib.md5() - m.update(ca_crt) + if ca_crt is not None: + m.update(ca_crt) m.update(admin_ep_cert) md5sum = m.hexdigest()