Merge "Mtce: Fix bmc password fetch error handling"

This commit is contained in:
Zuul 2022-06-01 15:51:12 +00:00 committed by Gerrit Code Review
commit 395f7ea5ec
10 changed files with 349 additions and 116 deletions

View File

@ -593,9 +593,16 @@ void httpUtil_handler ( struct evhttp_request *req, void *arg )
goto httpUtil_handler_done ;
}
httpUtil_handler_done:
/* Some handlers perform state transitions even in failure path.
* Allow the service handler to run during done stage if only
* for fault handling.
* Failing to allow a handler to detect its faults can be problematic.
*/
if ( event_ptr->handler )
{
// ilog ("%s calling event specific handler\n", event_ptr->log_prefix.c_str() );
rc = event_ptr->handler ( (*event_ptr) ) ;
}
else
@ -604,10 +611,6 @@ void httpUtil_handler ( struct evhttp_request *req, void *arg )
rc = event_ptr->status = FAIL_NULL_POINTER ;
}
httpUtil_handler_done:
// hlog2 ("%s Base:%p:%p Event:%p\n", event_ptr->log_prefix.c_str(), event_ptr->base, arg, event_ptr );
keyValObject.del_key ((unsigned long)arg );
event_ptr->active = false ;

View File

@ -17,6 +17,49 @@
#include <json-c/json.h> /* for ... json-c json string parsing */
#include <sstream>
/*********************************************************************
*
* The json-c shared library api allocates objects that it tracks by
* reference. Failure to help json-c manage its reference counts can
* lead to a process failure or memory leak.
*
* A process segmentation fault can occur if json_object_put is called
* with an invalid object or an object with 0 reference count.
*
* Failing to call json_object_put for json objects that increment
* the object's reference count will cause a memory leak.
*
* Calling json_object_put decrements the object's reference count.
*
* The following json api increments the allocated object's reference
* count requiring the caller to call json_object_put ( object )
* to decrement the reference count when a function is done with
* the object.
*
* json_object_new_object:
*
* This api creates a new empty object with a reference count 1.
* The caller of this object initially has sole ownership.
*
* json_tokener_parse:
*
* This function calls json_object_new_object behind the scenes,
* so you need to call json_object_put for the returned object.
*
* There is no need to adjust reference counts using json_object_put
* for these calls.
*
* json_object_object_get
* json_object_object_get_ex
* json_object_get_array
* json_object_to_json_string
*
* For more information refer to https://json-c.github.io/json-c
*
* centos uses json-c .11
* debian uses json-c .15
*
***********************************************************************/
using namespace std;
#ifdef __AREA__
@ -463,7 +506,6 @@ int jsonUtil_load_host ( char * json_str_ptr, node_inv_type & info )
/* init to null to avoid trap on early cleanup call with
* bad non-null default pointer value */
struct json_object *node_obj = (struct json_object *)(NULL);
struct json_object *err_obj = (struct json_object *)(NULL);
if (( json_str_ptr == NULL ) || ( *json_str_ptr == '\0' ) ||
( ! strncmp ( json_str_ptr, "(null)" , 6 )))
@ -548,7 +590,6 @@ int jsonUtil_load_host ( char * json_str_ptr, node_inv_type & info )
load_host_cleanup:
if (node_obj) json_object_put(node_obj);
if (err_obj) json_object_put(err_obj);
return (rc);
}
@ -563,7 +604,6 @@ int jsonUtil_load_host_state ( char * json_str_ptr, node_inv_type & info )
/* init to null to avoid trap on early cleanup call with
* bad non-null default pointer value */
struct json_object *node_obj = (struct json_object *)(NULL);
struct json_object *err_obj = (struct json_object *)(NULL);
if (( json_str_ptr == NULL ) || ( *json_str_ptr == '\0' ) ||
( ! strncmp ( json_str_ptr, "(null)" , 6 )))
@ -615,7 +655,6 @@ int jsonUtil_load_host_state ( char * json_str_ptr, node_inv_type & info )
load_host_cleanup:
if (node_obj) json_object_put(node_obj);
if (err_obj) json_object_put(err_obj);
return (rc);
}
@ -678,18 +717,18 @@ int jsonUtil_secret_load ( string & name,
else
{
elog ("%s Failed to find %s object array\n", name.c_str(), MTC_JSON_SECRET_LIST );
rc = FAIL_NOT_FOUND ;
}
}
else
{
elog ("%s Failed to find %s object\n", name.c_str(), MTC_JSON_SECRET_LIST );
rc = FAIL_NOT_FOUND ;
}
secret_load_cleanup:
if (raw_obj) json_object_put(raw_obj );
if (secret_obj) json_object_put(secret_obj );
if (ref_obj) json_object_put(ref_obj );
return (rc);
}
@ -1063,7 +1102,6 @@ int jsonApi_array_value ( char * json_str_ptr,
struct json_object *raw_obj = (struct json_object *)(NULL);
struct json_object *array_obj = (struct json_object *)(NULL);
struct json_object *tuple_obj = (struct json_object *)(NULL);
struct json_object *type_obj = (struct json_object *)(NULL);
if ( strlen(json_str_ptr) < 3 )
{
@ -1130,7 +1168,6 @@ int jsonApi_array_value ( char * json_str_ptr,
array_value_cleanup:
if (raw_obj) json_object_put(raw_obj);
if (type_obj) json_object_put(type_obj);
return (rc);
}

View File

@ -38,6 +38,27 @@
#include "jsonUtil.h" /* for ... Json utilities */
#include "secretUtil.h" /* this .. module header */
/**************************************************************************
*
* Name : secretUtil_manage_secret
*
* Description: This FSM serves to fetch a secret from barbican with error
* handling.
*
* The FSM uses a key value pair map list of type barbicanSecret_type
* defined in httpUtil.h
*
* typedef struct
* {
* mtc_secretStages_enum stage ;
* string reference;
* string payload ;
* } barbicanSecret_type;
*
* The fsm declares a c++ map of this type which serves as
* a key:value pair list keyed by host uuid string using the
* secretUtil_find_secret utility. */
std::map<string, barbicanSecret_type> secretList;
barbicanSecret_type * secretUtil_find_secret ( string & host_uuid )
@ -51,6 +72,46 @@ barbicanSecret_type * secretUtil_find_secret ( string & host_uuid )
return NULL;
}
/* The FSM uses mtc_secretStages_enum also defined in httpUtil.h
*
* MTC_SECRET__START ....... issue request to barbican for the list of
* secrets for this host_uuid.
*
* MTC_SECRET__GET_REF ..... wait for barbican's response with the
* reference uuid
*
* The secret_handler will store the secret reference uuid and change
* state to MTC_SECRET__GET_PWD or MTC_SECRET__GET_REF_FAIL if the
* request fails or there are no secrets available.
*
* MTC_SECRET__GET_PWD ..... issue a secret read request using the
* secret reference uuid
*
* MTC_SECRET__GET_PWD_REC . wait for barbican's response with the
* secret string and extract it once recv'ed
*
* The secret_handler will store the secret payload and change state
* to MTC_SECRET__GET_PWD_RECV or MTC_SECRET__GET_PWD_FAIL if the
* request to provide a secret payload failed or was empty.
*
* The secretUtil_get_secret and secretUtil_read_secret http requests
* are non-blocking.
*
* Parameters:
*
* event .......... reference to an http libevent event
* hostname ........ the host's name
* host_uuid ....... the host's uuid
* secret_timer .... a maintenance timer for request timeout detection
* secret_handler .. pointer
*
* Updates : secretList host_uuid key value is updated with reference
* and payload ; the secret.
*
* Returns : execution status
*
***************************************************************************/
barbicanSecret_type * secretUtil_manage_secret ( libEvent & event,
string & hostname,
string & host_uuid,
@ -64,100 +125,150 @@ barbicanSecret_type * secretUtil_manage_secret ( libEvent & event,
{
barbicanSecret_type secret;
secret.stage = MTC_SECRET__START;
secret.payload.clear();
secret.reference.clear();
it = secretList.insert( std::pair<string, barbicanSecret_type>( host_uuid, secret ) ).first;
}
if ( it->second.stage == MTC_SECRET__START )
if ( event.active == true )
{
it->second.reference.clear();
it->second.payload.clear();
if (( it->second.stage != MTC_SECRET__GET_REF ) &&
( it->second.stage != MTC_SECRET__GET_PWD ))
{
slog ("%s event is active while in the wrong stage (%d)", hostname.c_str(), it->second.stage );
event.active = false ; /* correct the error */
return & it->second ;
}
}
if ( it->second.stage == MTC_SECRET__START ||
it->second.stage == MTC_SECRET__GET_REF_FAIL )
switch ( it->second.stage )
{
if ( mtcTimer_expired ( secret_timer ) )
case MTC_SECRET__START:
{
rc = secretUtil_get_secret ( event, hostname, host_uuid );
if (rc)
if ( mtcTimer_expired ( secret_timer ) )
{
wlog ( "%s getting secret reference failed \n", host_uuid.c_str() );
it->second.stage = MTC_SECRET__GET_REF_FAIL;
mtcTimer_start( secret_timer, handler, SECRET_RETRY_DELAY );
ilog ("%s query bmc password", hostname.c_str());
it->second.reference.clear();
it->second.payload.clear();
rc = secretUtil_get_secret ( event, hostname, host_uuid );
if (rc)
{
elog ("%s get secret request failed (%d)", hostname.c_str(), rc );
it->second.stage = MTC_SECRET__GET_REF_FAIL;
}
else
{
it->second.stage = MTC_SECRET__GET_REF;
mtcTimer_start ( secret_timer, handler, SECRET_REPLY_DELAY );
}
}
break ;
}
case MTC_SECRET__GET_REF:
{
if ( mtcTimer_expired ( secret_timer ) == false )
{
if ( event.active == true )
{
/* Look for the response */
if ( event.base )
{
dlog ( "%s calling event_base_loop \n", hostname.c_str() );
event_base_loop ( event.base, EVLOOP_NONBLOCK );
}
else
{
/* should not get here. event active while base is null
* try and recover from this error case. */
slog ("%s active with null base", hostname.c_str());
event.active = false ;
}
}
}
else
{
mtcTimer_start( secret_timer, handler, SECRET_REPLY_DELAY );
elog ( "%s timeout waiting for secret reference \n", hostname.c_str() );
it->second.stage = MTC_SECRET__GET_REF_FAIL ;
}
break ;
}
else if ( event.base )
case MTC_SECRET__GET_PWD:
{
if ( mtcTimer_expired ( secret_timer ) == false )
{
if ( event.active == true )
{
/* Look for the response */
if ( event.base )
{
dlog ( "%s calling event_base_loop \n", hostname.c_str() );
event_base_loop( event.base, EVLOOP_NONBLOCK );
}
else
{
/* should not get here. event active while base is null
* try and recover from this error case. */
slog ("%s active with null base", hostname.c_str() );
event.active = false ;
}
}
}
else
{
elog ( "%s timeout waiting for secret payload \n", hostname.c_str() );
it->second.stage = MTC_SECRET__GET_PWD_FAIL ;
}
break ;
}
case MTC_SECRET__GET_REF_RECV:
{
mtcTimer_reset( secret_timer );
httpUtil_free_conn ( event );
httpUtil_free_base ( event );
}
}
else if ( it->second.stage == MTC_SECRET__GET_REF_RECV ||
it->second.stage == MTC_SECRET__GET_PWD_FAIL )
{
if ( mtcTimer_expired ( secret_timer ) )
{
rc = secretUtil_read_secret ( event, hostname, host_uuid );
if (rc)
{
wlog ( "%s getting secret payload failed \n", host_uuid.c_str() );
wlog ( "%s call to secretUtil_read_secret failed \n", hostname.c_str() );
it->second.stage = MTC_SECRET__GET_PWD_FAIL;
mtcTimer_start( secret_timer, handler, SECRET_RETRY_DELAY );
}
else
{
mtcTimer_start( secret_timer, handler, SECRET_REPLY_DELAY );
dlog ("%s waiting on secret password ; timeout in %d secs\n", hostname.c_str(), SECRET_REPLY_DELAY);
it->second.stage = MTC_SECRET__GET_PWD ;
mtcTimer_start ( secret_timer, handler, SECRET_REPLY_DELAY );
}
break ;
}
else if ( event.base )
case MTC_SECRET__GET_REF_FAIL:
case MTC_SECRET__GET_PWD_FAIL:
{
httpUtil_free_conn ( event );
httpUtil_free_base ( event );
}
}
else if ( it->second.stage == MTC_SECRET__GET_REF ||
it->second.stage == MTC_SECRET__GET_PWD )
{
if ( event.active == true )
{
/* Look for the response */
if ( event.base )
if ( it->second.stage == MTC_SECRET__GET_REF_FAIL )
{
event_base_loop( event.base, EVLOOP_NONBLOCK );
wlog ( "%s failed to get secret reference \n", hostname.c_str() );
}
else
{
/* should not get here. event active while base is null
* try and recover from this error case. */
event.active = false ;
wlog ( "%s failed to get secret payload \n", hostname.c_str() );
}
}
else if ( event.base )
{
if ( it->second.stage == MTC_SECRET__GET_REF )
{
wlog ( "%s getting secret reference timeout \n", host_uuid.c_str() );
it->second.stage = MTC_SECRET__GET_REF_FAIL ;
mtcTimer_reset( secret_timer );
mtcTimer_start( secret_timer, handler, SECRET_RETRY_DELAY );
}
if ( it->second.stage == MTC_SECRET__GET_PWD )
{
wlog ( "%s getting secret payload timeout \n", host_uuid.c_str() );
it->second.stage = MTC_SECRET__GET_PWD_FAIL ;
mtcTimer_reset( secret_timer );
mtcTimer_start( secret_timer, handler, SECRET_RETRY_DELAY );
}
it->second.stage = MTC_SECRET__START ;
mtcTimer_reset ( secret_timer );
mtcTimer_start ( secret_timer, handler, SECRET_RETRY_DELAY );
httpUtil_free_conn ( event );
httpUtil_free_base ( event );
break ;
}
}
default:
{
it->second.stage = MTC_SECRET__START ;
}
} // switch
return & it->second ;
}
@ -174,6 +285,7 @@ int secretUtil_get_secret ( libEvent & event,
string & hostname,
string & host_uuid )
{
dlog ("%s get secret", hostname.c_str());
httpUtil_event_init ( &event,
host_uuid,
"secretUtil_get_secret",
@ -230,6 +342,7 @@ int secretUtil_read_secret ( libEvent & event,
string & hostname,
string & host_uuid )
{
dlog ("%s read secret", hostname.c_str());
httpUtil_event_init ( &event,
host_uuid,
"secretUtil_read_secret",
@ -292,48 +405,53 @@ int secretUtil_handler ( libEvent & event )
jsonUtil_secret_type json_info ;
string hn = event.hostname ;
int rc = event.status ;
/* handler called */
event.active = false ;
std::map<string, barbicanSecret_type>::iterator it;
it = secretList.find( event.uuid );
if ( it == secretList.end() )
{
elog ("%s failed to find secret record\n", hn.c_str());
return ( rc ) ;
return 0 ;
}
if ( event.request == BARBICAN_GET_SECRET )
{
if ( event.status )
{
elog ("%s failed to get secret - error code (%d) \n", hn.c_str(), event.status );
it->second.stage = MTC_SECRET__GET_REF_FAIL;
return ( rc ) ;
elog ("%s failed to get secret ; %d\n", hn.c_str(), event.status) ;
return 0 ;
}
rc = jsonUtil_secret_load ( event.uuid,
(char*)event.response.data(),
int rc = jsonUtil_secret_load ( event.uuid,
(char*)event.response.data(),
json_info );
if ( rc != PASS )
{
elog ( "%s failed to parse secret response (%s)\n",
event.hostname.c_str(),
event.response.c_str() );
event.status = FAIL_JSON_PARSE ;
it->second.stage = MTC_SECRET__GET_REF_FAIL;
elog ("%s failed to parse secret response : %s (%d:%d)\n",
hn.c_str(),
event.response.c_str(),
rc, event.status );
}
else
{
size_t pos = json_info.secret_ref.find_last_of( '/' );
it->second.reference = json_info.secret_ref.substr( pos+1 );
if ( it->second.reference.empty() )
{
wlog ("%s no barbican secret reference found \n", hn.c_str() );
it->second.stage = MTC_SECRET__GET_PWD_RECV;
it->second.stage = MTC_SECRET__GET_REF_FAIL;
elog ("%s no barbican secret reference found : %s (%d)\n",
hn.c_str(),
event.response.c_str(),
FAIL_OPERATION ) ;
}
else
{
dlog ("%s barbican secret reference found \n", hn.c_str() );
it->second.stage = MTC_SECRET__GET_REF_RECV;
dlog ("%s barbican secret reference found\n", hn.c_str() ) ;
}
}
}
@ -341,22 +459,38 @@ int secretUtil_handler ( libEvent & event )
{
if ( event.status == HTTP_NOTFOUND )
{
wlog ("%s no barbican secret payload found \n", hn.c_str() );
it->second.stage = MTC_SECRET__GET_PWD_FAIL;
elog ("%s no barbican secret payload found : %s (rc:%d:%d)\n",
hn.c_str(),
event.response.c_str(),
event.status, FAIL_NOT_FOUND );
}
else if ( event.status != PASS )
{
elog ("%s failed to read secret - error code (%d) \n", hn.c_str(), event.status );
it->second.stage = MTC_SECRET__GET_REF_FAIL;
return ( rc ) ;
it->second.stage = MTC_SECRET__GET_PWD_FAIL;
elog ("%s failed to read secret : %s (rc:%d)\n",
hn.c_str(),
event.response.c_str(),
event.status );
}
else if ( event.response.empty() )
{
it->second.stage = MTC_SECRET__GET_PWD_FAIL;
wlog ("%s secret payload is empty (rc:%d)\n",
hn.c_str(),
FAIL_OPERATION ) ;
}
else
{
it->second.stage = MTC_SECRET__GET_PWD_RECV;
it->second.payload = event.response;
dlog ("%s secret payload : %s\n", hn.c_str(), event.response.c_str() );
}
dlog ("%s barbican secret payload found \n", hn.c_str() );
it->second.payload = event.response;
it->second.stage = MTC_SECRET__GET_PWD_RECV;
}
else
{
elog ("%s unsupported secret request (%d)\n", hn.c_str(), event.request );
slog ("%s unsupported secret request (rc:%d)\n", hn.c_str(), FAIL_BAD_CASE );
}
return ( rc ) ;
return ( 0 ) ;
}

View File

@ -47,8 +47,8 @@ using namespace std;
#define MTC_SECRET_PAYLOAD "payload" /**< barbican secret payload label */
#define SECRET_START_DELAY (1)
#define SECRET_REPLY_DELAY (1)
#define SECRET_RETRY_DELAY (8)
#define SECRET_REPLY_DELAY (10)
#define SECRET_RETRY_DELAY (10)
barbicanSecret_type * secretUtil_find_secret ( string & host_uuid );
barbicanSecret_type * secretUtil_manage_secret ( libEvent & event,

View File

@ -96,6 +96,7 @@ void daemon_healthcheck ( const char * sig );
void daemon_health_test ( void );
bool daemon_is_file_present ( const char * filename );
bool daemon_is_os_debian ( void );
int daemon_get_rmem_max ( void );
typedef struct

View File

@ -52,6 +52,8 @@ extern char *program_invocation_short_name;
static char pid_filename [MAX_FILENAME_LEN] ;
static char hc_filename [MAX_FILENAME_LEN] ;
#define BUFFER 1024
void daemon_files_fini ( void )
{
close_syslog();
@ -64,6 +66,50 @@ void daemon_health_test ( void )
daemon_healthcheck (&file_str[0]);
}
bool daemon_is_os_debian ( void )
{
#define OS_RELEASE_FILE ((const char *)("/etc/os-release"))
char buffer [BUFFER];
char line [BUFFER];
memset ( line, 0, BUFFER );
FILE * file_stream = fopen ( OS_RELEASE_FILE, "r" );
if ( file_stream != NULL )
{
int rc ;
while ( fgets (buffer, BUFFER, file_stream) != NULL )
{
rc = sscanf ( &buffer[0], "ID=%1023s", &line[0] );
if ( rc == 1 )
{
string os = line ;
if ( ! os.compare("debian") )
{
dlog("%s:%s", OS_RELEASE_FILE, os.c_str());
/* Close the file stream */
fclose(file_stream);
return ( true ) ;
}
}
else
{
dlog1 ("%s:%s", OS_RELEASE_FILE, &line[0]);
}
}
/* Close the file stream */
fclose(file_stream);
}
else
{
elog ("failed to open %s", OS_RELEASE_FILE);
}
return ( false );
}
bool daemon_is_file_present ( const char * filename )
{
struct stat p ;
@ -102,7 +148,6 @@ void daemon_healthcheck ( const char * sig )
}
}
#define BUFFER 1024
int daemon_log_value ( const char * filename , const char * str, int val )
{

View File

@ -1232,11 +1232,11 @@ int hwmonHostClass::bmc_sensor_monitor ( struct hwmonHostClass::hwmon_host * hos
/* NOTE: This parsing method is not leaking memory ; verified ! */
json_bool status ;
struct json_object * req_obj = (struct json_object *)(NULL) ;
struct json_object * raw_obj = json_tokener_parse( host_ptr->bmc_thread_info.data.data() );
if ( raw_obj )
{
/* Look for ... BMC_JSON__SENSOR_DATA_MESSAGE_HEADER */
struct json_object * req_obj = (struct json_object *)(NULL) ;
status = json_object_object_get_ex ( raw_obj, BMC_JSON__SENSOR_DATA_MESSAGE_HEADER, &req_obj );
if (( status == true ) && req_obj )
{
@ -1286,7 +1286,6 @@ int hwmonHostClass::bmc_sensor_monitor ( struct hwmonHostClass::hwmon_host * hos
}
if (raw_obj) json_object_put(raw_obj);
if (req_obj) json_object_put(req_obj);
}
if ( host_ptr->bmc_thread_info.status )

View File

@ -1660,11 +1660,14 @@ int hwmonHttp_add_sensor ( string & hostname,
event.payload.append ("\",\"audit_interval\":") ;
event.payload.append ("0");
event.payload.append (",\"suppress\":\"") ;
if ( sensor.suppress == true )
event.payload.append ("True\"");
else
event.payload.append ("False\"");
if ( daemon_is_os_debian () == false )
{
event.payload.append (",\"suppress\":\"") ;
if ( sensor.suppress == true )
event.payload.append ("True\"");
else
event.payload.append ("False\"");
}
event.payload.append ("}");
@ -1843,11 +1846,14 @@ int hwmonHttp_add_group ( string & hostname,
event.payload.append ("\",\"audit_interval_group\":") ;
event.payload.append (itos(sensor_group.group_interval));
event.payload.append (",\"suppress\":\"") ;
if ( sensor_group.suppress == true )
event.payload.append ("True\"");
else
event.payload.append ("False\"");
if ( daemon_is_os_debian () == false )
{
event.payload.append (",\"suppress\":\"") ;
if ( sensor_group.suppress == true )
event.payload.append ("True\"");
else
event.payload.append ("False\"");
}
event.payload.append ("}");

View File

@ -56,7 +56,6 @@ int hwmonJson_load_inv ( char * json_str_ptr, node_inv_type & info )
/* init to null to avoid trap on early cleanup call with
* bad non-null default pointer value */
struct json_object *node_obj = (struct json_object *)(NULL);
struct json_object *err_obj = (struct json_object *)(NULL);
if (( json_str_ptr == NULL ) || ( *json_str_ptr == '\0' ) ||
( ! strncmp ( json_str_ptr, "(null)" , 6 )))
@ -99,7 +98,6 @@ int hwmonJson_load_inv ( char * json_str_ptr, node_inv_type & info )
hwmon_info_cleanup:
if (node_obj) json_object_put(node_obj);
if (err_obj) json_object_put(err_obj);
return (rc);
}

View File

@ -6508,9 +6508,19 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr )
mtcTimer_handler );
if ( secret->stage == MTC_SECRET__GET_PWD_RECV )
{
node_ptr->bm_pw = secret->payload ;
ilog ("%s bmc credentials received",
node_ptr->hostname.c_str());
httpUtil_free_conn ( node_ptr->secretEvent );
httpUtil_free_base ( node_ptr->secretEvent );
if ( secret->payload.empty() )
{
wlog ("%s failed to acquire bmc password", node_ptr->hostname.c_str());
secret->stage = MTC_SECRET__GET_PWD_FAIL ;
}
else
{
node_ptr->bm_pw = secret->payload ;
ilog ("%s bmc password received", node_ptr->hostname.c_str());
secret->stage = MTC_SECRET__START ;
}
}
else
{