From aeb2c1f20af84047803533de1873d8887722501b Mon Sep 17 00:00:00 2001 From: Alex Kozyrev Date: Thu, 4 Apr 2019 09:14:27 -0400 Subject: [PATCH] Fix for MTCE race condition in BMC secret handling There is intermittent issue in getting BMC password in MTCE. The process of obtaining a secret from Barbican stops after a secret reference is received. No attempts to retrieve the actual payload is atempted. This happens when the secret reference reply is received right after BMC queries are initiated. It was fine before when we had an one-stage process of getting a password from keyring. We cannot allow it now because of a two-stage Barbican process. Change-Id: I381f69ab6a1a54118b22dd31feefcd93698120ad Closes-bug: 1818284 Signed-off-by: Alex Kozyrev --- mtce-common/src/common/secretUtil.cpp | 9 +++++++-- mtce/src/hwmon/hwmonFsm.cpp | 2 +- mtce/src/maintenance/mtcNodeHdlrs.cpp | 29 +++++++++++---------------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mtce-common/src/common/secretUtil.cpp b/mtce-common/src/common/secretUtil.cpp index c92251cb..5807691f 100755 --- a/mtce-common/src/common/secretUtil.cpp +++ b/mtce-common/src/common/secretUtil.cpp @@ -51,7 +51,6 @@ barbicanSecret_type * secretUtil_find_secret ( string & host_uuid ) return NULL; } - barbicanSecret_type * secretUtil_manage_secret ( libEvent & event, string & host_uuid, struct mtc_timer & secret_timer, @@ -142,14 +141,18 @@ barbicanSecret_type * secretUtil_manage_secret ( libEvent & event, { wlog ( "%s getting secret reference timeout \n", host_uuid.c_str() ); it->second.stage = MTC_SECRET__GET_REF_FAIL ; + mtcTimer_reset( secret_timer ); mtcTimer_start( secret_timer, handler, SECRET_RETRY_DELAY ); + } if ( it->second.stage == MTC_SECRET__GET_PWD ) { wlog ( "%s getting secret payload timeout \n", host_uuid.c_str() ); it->second.stage = MTC_SECRET__GET_PWD_FAIL ; + mtcTimer_reset( secret_timer ); mtcTimer_start( secret_timer, handler, SECRET_RETRY_DELAY ); } + httpUtil_free_conn ( event ); httpUtil_free_base ( event ); } @@ -221,7 +224,7 @@ int secretUtil_read_secret ( libEvent & event, string & host_uuid ) { httpUtil_event_init ( &event, host_uuid, - "secretUtil_get_secret", + "secretUtil_read_secret", hostUtil_getServiceIp (SERVICE_SECRET), hostUtil_getServicePort(SERVICE_SECRET)); @@ -320,6 +323,7 @@ int secretUtil_handler ( libEvent & event ) } else { + ilog ("%s barbican secret reference found \n", hn.c_str() ); it->second.stage = MTC_SECRET__GET_REF_RECV; } } @@ -337,6 +341,7 @@ int secretUtil_handler ( libEvent & event ) return ( rc ) ; } + ilog ("%s barbican secret payload found \n", hn.c_str() ); it->second.payload = event.response; it->second.stage = MTC_SECRET__GET_PWD_RECV; } diff --git a/mtce/src/hwmon/hwmonFsm.cpp b/mtce/src/hwmon/hwmonFsm.cpp index 789a298d..52a50194 100644 --- a/mtce/src/hwmon/hwmonFsm.cpp +++ b/mtce/src/hwmon/hwmonFsm.cpp @@ -153,7 +153,7 @@ void hwmonHostClass::hwmon_fsm ( void ) host_ptr->thread_extra_info.bm_pw = "" ; } #endif - if (( host_ptr->thread_extra_info.bm_pw.empty ()) && ( host_ptr->ping_info.ok == true )) + if ( host_ptr->thread_extra_info.bm_pw.empty () ) { string host_uuid = hostBase.get_uuid(host_ptr->hostname); wlog_throttled ( host_ptr->empty_secret_log_throttle, 20, diff --git a/mtce/src/maintenance/mtcNodeHdlrs.cpp b/mtce/src/maintenance/mtcNodeHdlrs.cpp index 0fa5a428..590fd831 100755 --- a/mtce/src/maintenance/mtcNodeHdlrs.cpp +++ b/mtce/src/maintenance/mtcNodeHdlrs.cpp @@ -5831,7 +5831,17 @@ int nodeLinkClass::bm_handler ( struct nodeLinkClass::node * node_ptr ) mtcTimer_start ( node_ptr->bmc_access_timer, mtcTimer_handler, MTC_MINS_2 ); } - if (( node_ptr->thread_extra_info.bm_pw.empty ()) && ( node_ptr->bm_ping_info.ok == true )) + if ( node_ptr->bm_ping_info.ok == false ) + { + /* Auto correct key ping information ; should ever occur but if it does ... */ + if (( node_ptr->bm_ping_info.hostname.empty()) || ( node_ptr->bm_ping_info.ip.empty())) + { + node_ptr->bm_ping_info.hostname = node_ptr->hostname ; + node_ptr->bm_ping_info.ip = node_ptr->bm_ip ; + } + } + + if ( node_ptr->thread_extra_info.bm_pw.empty() ) { barbicanSecret_type * secret = secretUtil_manage_secret( node_ptr->secretEvent, node_ptr->uuid, @@ -5842,9 +5852,8 @@ int nodeLinkClass::bm_handler ( struct nodeLinkClass::node * node_ptr ) node_ptr->thread_extra_info.bm_pw = node_ptr->bm_pw = secret->payload ; } } - /* This block queries and logs BMC Info and last Reset Cause */ - if (( node_ptr->bm_accessible == false ) && + else if (( node_ptr->bm_accessible == false ) && ( node_ptr->bm_ping_info.ok == true ) && (( node_ptr->mc_info_query_done == false ) || ( node_ptr->reset_cause_query_done == false ) || @@ -6004,20 +6013,6 @@ int nodeLinkClass::bm_handler ( struct nodeLinkClass::node * node_ptr ) } } } - if ( node_ptr->bm_ping_info.ok == false ) - { - /* Auto correct key ping information ; should ever occur but if it does ... */ - if (( node_ptr->bm_ping_info.hostname.empty()) || ( node_ptr->bm_ping_info.ip.empty())) - { - /* if the bm ip is not yet learned then this log will flood */ - //slog ("%s host ping info missing ; (%d:%d)\n", - // node_ptr->hostname.c_str(), - // node_ptr->bm_ping_info.hostname.empty(), - // node_ptr->bm_ping_info.ip.empty()); - node_ptr->bm_ping_info.hostname = node_ptr->hostname ; - node_ptr->bm_ping_info.ip = node_ptr->bm_ip ; - } - } /* don't run the ping monitor if the ip address is invalid */ if ( hostUtil_is_valid_ip_addr ( node_ptr->bm_ping_info.ip ) == true )