From 0d63a16d8d2f3c807f2e46b2b8dc5123328bc4b0 Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Thu, 12 Sep 2019 09:59:13 -0400 Subject: [PATCH] Improve BMC password first fetch handling in hwmon Trying to get the BMC password through barbican before the ping succeeds leads to an early bmc access lost failure that 1. produces a misleading bmc access lost failure log ; bmc access had not even been established yet. 2. imposes as retry wait that delays re-establishing bmc access and therefore overall sensor monitoring. This update also 1. adds hostname to some of the secretUtil API interfaces so that logs ar reported against the correct host rather than always the current controller hostname. 2. Changes some success path logging to dlogs to reduce log noise. 3. simplifies a ping ok log Change-Id: Ib3b7de212294d6dc350ee17d363f4009b3b0dcb0 Story: 2005861 Task: 36595 Signed-off-by: Eric MacDonald --- mtce-common/src/common/pingUtil.cpp | 9 ++---- mtce-common/src/common/secretUtil.cpp | 29 +++++++++++------- mtce-common/src/common/secretUtil.h | 5 ++-- mtce/src/hwmon/hwmonFsm.cpp | 42 ++++++++++++++------------- mtce/src/maintenance/mtcNodeHdlrs.cpp | 1 + 5 files changed, 47 insertions(+), 39 deletions(-) diff --git a/mtce-common/src/common/pingUtil.cpp b/mtce-common/src/common/pingUtil.cpp index c51c41ac..4d1a72d1 100644 --- a/mtce-common/src/common/pingUtil.cpp +++ b/mtce-common/src/common/pingUtil.cpp @@ -504,14 +504,11 @@ int pingUtil_recv ( ping_info_type & ping_info, * are in monitoring mode. */ if ( ping_info.monitoring == false ) { - /* ... only want the log when we ar first connecting */ - ilog ("%s ping recv %s ok ; identity:%04x sequence:%04x (try %d) (%d)\n", + /* ... only want the log when we are first connecting */ + ilog ("%s ping to %s ok ; (try %d)", ping_info.hostname.c_str(), ping_info.ip.c_str(), - ping_info.identity, - ping_info.sequence, - ping_info.recv_retries+1, - i); + ping_info.recv_retries+1); } else { diff --git a/mtce-common/src/common/secretUtil.cpp b/mtce-common/src/common/secretUtil.cpp index 5807691f..a0ffc926 100755 --- a/mtce-common/src/common/secretUtil.cpp +++ b/mtce-common/src/common/secretUtil.cpp @@ -52,6 +52,7 @@ barbicanSecret_type * secretUtil_find_secret ( string & host_uuid ) } barbicanSecret_type * secretUtil_manage_secret ( libEvent & event, + string & hostname, string & host_uuid, struct mtc_timer & secret_timer, void (*handler)(int, siginfo_t*, void*)) @@ -77,7 +78,7 @@ barbicanSecret_type * secretUtil_manage_secret ( libEvent & event, { if ( secret_timer.ring == true ) { - rc = secretUtil_get_secret ( event, host_uuid ); + rc = secretUtil_get_secret ( event, hostname, host_uuid ); if (rc) { wlog ( "%s getting secret reference failed \n", host_uuid.c_str() ); @@ -100,7 +101,7 @@ barbicanSecret_type * secretUtil_manage_secret ( libEvent & event, { if ( secret_timer.ring == true ) { - rc = secretUtil_read_secret ( event, host_uuid ); + rc = secretUtil_read_secret ( event, hostname, host_uuid ); if (rc) { wlog ( "%s getting secret payload failed \n", host_uuid.c_str() ); @@ -169,7 +170,9 @@ barbicanSecret_type * secretUtil_manage_secret ( libEvent & event, * */ -int secretUtil_get_secret ( libEvent & event, string & host_uuid ) +int secretUtil_get_secret ( libEvent & event, + string & hostname, + string & host_uuid ) { httpUtil_event_init ( &event, host_uuid, @@ -185,11 +188,13 @@ int secretUtil_get_secret ( libEvent & event, string & host_uuid ) } else { - elog ("%s failed to find secret record\n", host_uuid.c_str()); + elog ("%s failed to find secret record (%s)\n", + hostname.c_str(), + host_uuid.c_str()); return FAIL; } - event.hostname = _hn(); + event.hostname = hostname ; event.uuid = host_uuid; event.token.url = MTC_SECRET_LABEL; @@ -220,7 +225,9 @@ int secretUtil_get_secret ( libEvent & event, string & host_uuid ) * *********************************************************************/ -int secretUtil_read_secret ( libEvent & event, string & host_uuid ) +int secretUtil_read_secret ( libEvent & event, + string & hostname, + string & host_uuid ) { httpUtil_event_init ( &event, host_uuid, @@ -242,7 +249,7 @@ int secretUtil_read_secret ( libEvent & event, string & host_uuid ) return FAIL; } - event.hostname = _hn(); + event.hostname = hostname ; event.uuid = host_uuid; event.token.url = MTC_SECRET_LABEL; @@ -318,12 +325,12 @@ int secretUtil_handler ( libEvent & event ) it->second.reference = json_info.secret_ref.substr( pos+1 ); if ( it->second.reference.empty() ) { - ilog ("%s no barbican secret reference found \n", hn.c_str() ); + wlog ("%s no barbican secret reference found \n", hn.c_str() ); it->second.stage = MTC_SECRET__GET_PWD_RECV; } else { - ilog ("%s barbican secret reference found \n", hn.c_str() ); + dlog ("%s barbican secret reference found \n", hn.c_str() ); it->second.stage = MTC_SECRET__GET_REF_RECV; } } @@ -332,7 +339,7 @@ int secretUtil_handler ( libEvent & event ) { if ( event.status == HTTP_NOTFOUND ) { - ilog ("%s no barbican secret payload found \n", hn.c_str() ); + wlog ("%s no barbican secret payload found \n", hn.c_str() ); } else if ( event.status != PASS ) { @@ -341,7 +348,7 @@ int secretUtil_handler ( libEvent & event ) return ( rc ) ; } - ilog ("%s barbican secret payload found \n", hn.c_str() ); + dlog ("%s barbican secret payload found \n", hn.c_str() ); it->second.payload = event.response; it->second.stage = MTC_SECRET__GET_PWD_RECV; } diff --git a/mtce-common/src/common/secretUtil.h b/mtce-common/src/common/secretUtil.h index c222da66..f23e83bc 100755 --- a/mtce-common/src/common/secretUtil.h +++ b/mtce-common/src/common/secretUtil.h @@ -52,12 +52,13 @@ using namespace std; barbicanSecret_type * secretUtil_find_secret ( string & host_uuid ); barbicanSecret_type * secretUtil_manage_secret ( libEvent & event, + string & hostname, string & host_uuid, struct mtc_timer & secret_timer, void (*handler)(int, siginfo_t*, void*) ); int secretUtil_handler ( libEvent & event ); -int secretUtil_get_secret ( libEvent & event, string & host_uuid ); -int secretUtil_read_secret ( libEvent & event, string & host_uuid ); +int secretUtil_get_secret ( libEvent & event, string & hostname, string & host_uuid ); +int secretUtil_read_secret ( libEvent & event, string & hostname, string & host_uuid ); #endif /* __INCLUDE_MTCSECRETUTIL_H__ */ diff --git a/mtce/src/hwmon/hwmonFsm.cpp b/mtce/src/hwmon/hwmonFsm.cpp index e5e7734b..79fe8e40 100644 --- a/mtce/src/hwmon/hwmonFsm.cpp +++ b/mtce/src/hwmon/hwmonFsm.cpp @@ -98,10 +98,10 @@ void hwmonHostClass::hwmon_fsm ( void ) /* Check to see if sensor monitoring for this host is disabled. * If it is ... */ - if ( host_ptr->monitor == false ) + if (( host_ptr->monitor == false ) || ( host_ptr->bm_pw.empty())) { /* ... make sure the thread sits in the - * idle state while disabled */ + * idle state while disabled or there is no pw learned yet */ if ( thread_idle ( host_ptr->bmc_thread_ctrl ) == false ) { if ( thread_done ( host_ptr->bmc_thread_ctrl ) == true ) @@ -113,10 +113,28 @@ void hwmonHostClass::hwmon_fsm ( void ) thread_kill ( host_ptr->bmc_thread_ctrl, host_ptr->bmc_thread_info ); } } + if ( host_ptr->bm_pw.empty( )) + { + string host_uuid = hostBase.get_uuid(host_ptr->hostname); + + barbicanSecret_type * secret = + secretUtil_manage_secret( host_ptr->secretEvent, + host_ptr->hostname, + host_uuid, + host_ptr->secretTimer, + hwmonTimer_handler ); + + if ( secret->stage == MTC_SECRET__GET_PWD_RECV ) + { + host_ptr->thread_extra_info.bm_pw = host_ptr->bm_pw = secret->payload ; + } + ilog_throttled (host_ptr->empty_secret_log_throttle, 50, + "%s waiting on bm password learn", host_ptr->hostname.c_str()); + } continue ; } - if (( host_ptr->accessible == false ) && ( host_ptr->ping_info.ok == true )) + else if (( host_ptr->accessible == false ) && ( host_ptr->ping_info.ok == true )) { ilog ("%s bmc is accessible\n", host_ptr->hostname.c_str()); host_ptr->accessible = true ; @@ -153,23 +171,7 @@ void hwmonHostClass::hwmon_fsm ( void ) host_ptr->thread_extra_info.bm_pw = "" ; } #endif - if ( host_ptr->thread_extra_info.bm_pw.empty () ) - { - string host_uuid = hostBase.get_uuid(host_ptr->hostname); - wlog_throttled ( host_ptr->empty_secret_log_throttle, 20, - "%s bm password is empty ; learning and forcing reconnect\n", - host_ptr->hostname.c_str()); - barbicanSecret_type * secret = secretUtil_manage_secret( host_ptr->secretEvent, - host_uuid, - host_ptr->secretTimer, - hwmonTimer_handler ); - if ( secret->stage == MTC_SECRET__GET_PWD_RECV ) - { - host_ptr->ping_info.ok = false ; - host_ptr->thread_extra_info.bm_pw = host_ptr->bm_pw = secret->payload ; - } - } - else if ( host_ptr->accessible ) + if ( host_ptr->accessible ) { /* typical success path */ hwmonHostClass::bmc_sensor_monitor ( host_ptr ); diff --git a/mtce/src/maintenance/mtcNodeHdlrs.cpp b/mtce/src/maintenance/mtcNodeHdlrs.cpp index 714b4c28..1812849b 100755 --- a/mtce/src/maintenance/mtcNodeHdlrs.cpp +++ b/mtce/src/maintenance/mtcNodeHdlrs.cpp @@ -6222,6 +6222,7 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr ) if ( node_ptr->thread_extra_info.bm_pw.empty() ) { barbicanSecret_type * secret = secretUtil_manage_secret( node_ptr->secretEvent, + node_ptr->hostname, node_ptr->uuid, node_ptr->bm_timer, mtcTimer_handler );