From 3fbe5f1aa6ed2f9d8242e62e0e5d07c02e7ae378 Mon Sep 17 00:00:00 2001 From: Andy Ning Date: Tue, 12 Mar 2024 10:53:53 -0400 Subject: [PATCH] Add IPsec certificates renewal cron job This change added the IPsec certificates renewal script, and set it up as a cron job to run daily at mid night. Test Plan: PASS: After a DX system deployed, verify the script is in the correct directory with right permission, and is added in /var/spool/cron/crontabs/root PASS: Simulate the IPsec cert is about to expire, run the script, verify IPsec cert, private key and trusted CA cert are renewed, and IKE SAs and CHILD SAs are re-established. PASS: Simulate a failure condition (eg, ipsec-client return non zero), run the script, verify the IPsec renewal fails, and alarm 250.004 is raised. PASS: Run the script with IPsec cert not being about to expire, verify the script finish successfully and alarm 250.004 is cleared. PASS: Simulate the IPsec trusted CA cert is different from the system-local-ca in k8s secret, run the script, verify the trusted CA and IPsec cert/key are renewed, and IKE SAs and CHILD SAs are re-established. Story: 2010940 Task: 49705 Depends-On: https://review.opendev.org/c/starlingx/fault/+/912598 Change-Id: I69236399b59655dd67ac7b01c4472a4b7ab911e5 Signed-off-by: Andy Ning --- sysinv/sysinv/debian/deb_folder/rules | 2 + .../sysinv/debian/deb_folder/sysinv.install | 1 + .../sysinv/sysinv/scripts/ipsec-cert-renew.sh | 124 ++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 sysinv/sysinv/sysinv/scripts/ipsec-cert-renew.sh diff --git a/sysinv/sysinv/debian/deb_folder/rules b/sysinv/sysinv/debian/deb_folder/rules index 1f4767bc3a..d541b823aa 100755 --- a/sysinv/sysinv/debian/deb_folder/rules +++ b/sysinv/sysinv/debian/deb_folder/rules @@ -25,6 +25,7 @@ override_dh_install: install -p -D -m 700 $(CURDIR)/etc/sysinv/delete_load.sh $(CURDIR)/debian/tmp/etc/sysinv/upgrades/delete_load.sh install -p -D -m 644 debian/tmpfiles.conf $(CURDIR)/debian/tmp/usr/lib/tmpfiles.d/sysinv.conf install -p -D -m 700 $(CURDIR)/scripts/kube-cert-rotation.sh $(CURDIR)/debian/tmp/usr/bin/kube-cert-rotation.sh + install -p -D -m 700 $(CURDIR)/scripts/ipsec-cert-renew.sh $(CURDIR)/debian/tmp/usr/bin/ipsec-cert-renew.sh dh_install override_dh_python3: @@ -36,3 +37,4 @@ override_dh_installsystemd: override_dh_fixperms: dh_fixperms -Xkube-cert-rotation.sh + dh_fixperms -Xipsec-cert-renew.sh diff --git a/sysinv/sysinv/debian/deb_folder/sysinv.install b/sysinv/sysinv/debian/deb_folder/sysinv.install index a787072178..da49708364 100644 --- a/sysinv/sysinv/debian/deb_folder/sysinv.install +++ b/sysinv/sysinv/debian/deb_folder/sysinv.install @@ -18,6 +18,7 @@ usr/bin/cert-mon usr/bin/ipsec-server usr/bin/ipsec-client usr/bin/kube-cert-rotation.sh +usr/bin/ipsec-cert-renew.sh usr/bin/sysinv-agent usr/bin/sysinv-api usr/bin/sysinv-app diff --git a/sysinv/sysinv/sysinv/scripts/ipsec-cert-renew.sh b/sysinv/sysinv/sysinv/scripts/ipsec-cert-renew.sh new file mode 100644 index 0000000000..1df8ede0fd --- /dev/null +++ b/sysinv/sysinv/sysinv/scripts/ipsec-cert-renew.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) 2024 Wind River Systems, Inc. +# + +# +# This script checks validation of IPsec certificates, and call +# ipsec-client to renew them if necessary. +# + +# Renew certificates 15 days before expiration +declare -r CUTOFF_DAYS=15 +declare -r CUTOFF_DAYS_S=$((${CUTOFF_DAYS}*24*3600)) + +NAME=$(basename $0) +KUBE_CONFIG=/etc/kubernetes/admin.conf +IPSEC_CERT_DIR=/etc/swanctl/x509 +IPSEC_CERT_PATH="$IPSEC_CERT_DIR/system-ipsec-certificate-${HOSTNAME}.crt" +#IPSEC_CERT_PATH="$IPSEC_CERT_DIR/system-ipsec-certificate-${HOSTNAME}.crt-fake" +ERR_CA=0 +ERR_CERT=0 +ERR_RENEW=0 +RENEWAL_REQUIRED=0 + +# Log info message to /var/log/cron.log +function LOG_info { + logger -p cron.info -t "${NAME}($$): " "${@}" +} + +# Log error message to /var/log/cron.log +function LOG_error { + logger -p cron.error -t "${NAME}($$): " "${@}" +} + +# Retrieve a certificate's valid time by openssl +time_left_s_by_openssl() { + local time_left_s="" + local exp_date="" + exp_date=$(openssl x509 -in "$1" -enddate -noout| awk -F"=" '{print $2}') + if [ "x${exp_date}" != "x" ]; then + exp_date_s=$(date -d "${exp_date}" +%s) + current_date_s=$(date +%s) + time_left_s=$((${exp_date_s}-${current_date_s})) + else + return 1 + fi +} + +# Check if the trusted CA cert is consistent with system-local-ca +# in kubernetes. If it's not consistent, call ipsec-client to renew. +# This is for cases such as a node is offline and misses a system-local-ca +# update. + +# Retrieve the serial number of system-local-ca cert. +if [ ${ERR_CA} -eq 0 ]; then + serial_in_secret=$(kubectl --kubeconfig=/etc/kubernetes/admin.conf get secret system-local-ca -n cert-manager -o jsonpath='{.data.tls\.crt}' | base64 --decode | openssl x509 -noout -serial) + + if [ "x${serial_in_secret}" = "x" ]; then + LOG_error "Failed to retrieve system-local-ca from secret." + ERR_CA=1 + fi +fi + +# Retrieve the serial number of the IPsec trusted CA cert. +if [ ${ERR_CA} -eq 0 ]; then + serial_in_file=$(openssl x509 -in /etc/swanctl/x509ca/system-local-ca.crt -noout -serial) + + if [ "x${serial_in_file}" = "x" ]; then + LOG_error "Failed to retrieve serial number from CA cert file." + ERR_CA=1 + fi +fi + +# Compare to decide if they are consistent. +if [ ${ERR_CA} -eq 0 ]; then + if [ "${serial_in_secret}" != "${serial_in_file}" ]; then + LOG_info "IPsec trusted CA is diverse from system-local-ca." + RENEWAL_REQUIRED=1 + fi +fi + +# Check if it's time to renew IPsec certificate. +if [ ${ERR_CERT} -eq 0 ]; then + time_left_s=$(time_left_s_by_openssl "${IPSEC_CERT_PATH}") + if [ $? -ne 0 ]; then + LOG_error "Failed to retrieve expiry date from ${IPSEC_CERT_PATH}" + ERR_CERT=1 + fi +fi + +if [ ${ERR_CERT} -eq 0 ]; then + if [ "${time_left_s}" -lt "${CUTOFF_DAYS_S}" ]; then + LOG_info "IPsec certificate will expire in ${time_left_s}s, will be renewed." + RENEWAL_REQUIRED=1 + fi +fi + +# Call ipsec-client to renew IPsec certificates if trusted CA and/or +# IPsec cert renewal is required. +if [ $RENEWAL_REQUIRED -eq 1 ]; then + ipsec-client -o 2 pxecontroller + if [ $? -ne 0 ]; then + LOG_error "ipsec-client failed to renew IPsec certificates." + ERR_RENEW=1 + else + LOG_info "IPsec certificate successfully renewed." + fi +else + if [ ${ERR_CA} -ne 0 ] || [ ${ERR_CERT} -ne 0 ]; then + ERR_RENEW=1 + fi +fi + +# Raise alarm if anyting goes wrong. +if [ ${ERR_RENEW} -ne 0 ]; then + /usr/local/bin/fmClientCli -c "### ###250.004###set###host###host=${HOSTNAME}### ###major###IPsec certificates renewal failed.###operational-violation### ###Check cron.log and ipsec-auth.log, fix the issue and rerun $NAME.### ### ###" +else + /usr/local/bin/fmClientCli -A "250.004" &> /dev/null + if [ $? -eq 0 ]; then + /usr/local/bin/fmClientCli -d "###250.004###host=${HOSTNAME}###" + fi +fi