From 4d2383818f41552e87ad64bb25798a7c264639c5 Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Thu, 29 Aug 2019 08:55:28 -0400 Subject: [PATCH] Add bmc protocol select to maintenance This update adds BMC Info Query command handling and info logging to maintenance. Example of the logs produced by the BMC Query are compute-2 manufacturer is Intel Corporation compute-2 model number: part number: serial number: compute-2 BIOS firmware version is SE5C620.86B.00.01.0013.030920180427 compute-2 BMC firmware version is unavailable compute-2 power is on compute-2 has 2 processors compute-2 has 192 GiB of memory Please note that the default protocol remains IPMI even if Redfish support is detected. This is because the power/reset/netboot control implementation for Redfish has not yet been implemented. Test Plan: PASS: Verify redfish BMC info query logging. PASS: Verify IPMI remains the default selected protocol. Regression: PASS: Verify sensor monitoring and alarming still works. PASS: Verify power-off command handling. PASS: Verify power-on command handling. PASS: Verify reset command handling. PASS: Verify reboot command handling. PASS: Verify reinstall (netboot) command handling. Change-Id: I654056119018a1751a70495e3df8b541d9e00b93 Story: 2005861 Task: 35826 Signed-off-by: Eric MacDonald --- mtce-common/src/common/bmcUtil.cpp | 64 ++++++- mtce-common/src/common/bmcUtil.h | 43 ++++- mtce-common/src/common/ipmiUtil.cpp | 4 +- mtce-common/src/common/redfishUtil.cpp | 128 +++++++++++++- mtce-common/src/common/redfishUtil.h | 39 ++++- mtce-common/src/common/threadUtil.cpp | 2 +- mtce-common/src/daemon/daemon_files.cpp | 2 +- mtce/src/common/nodeClass.cpp | 50 +++--- mtce/src/heartbeat/hbsStubs.cpp | 11 ++ mtce/src/maintenance/mtcBmcUtil.cpp | 23 +-- mtce/src/maintenance/mtcCtrlMsg.cpp | 5 +- mtce/src/maintenance/mtcNodeHdlrs.cpp | 212 ++++++++++++++++++------ mtce/src/maintenance/mtcThreads.cpp | 9 +- 13 files changed, 491 insertions(+), 101 deletions(-) diff --git a/mtce-common/src/common/bmcUtil.cpp b/mtce-common/src/common/bmcUtil.cpp index 22a68829..6164b5fc 100644 --- a/mtce-common/src/common/bmcUtil.cpp +++ b/mtce-common/src/common/bmcUtil.cpp @@ -87,8 +87,8 @@ string bmcUtil_getProtocol_str ( bmc_protocol_enum protocol ) { switch (protocol) { - case BMC_PROTOCOL__REDFISHTOOL: return("redfishtool"); - case BMC_PROTOCOL__IPMITOOL: return("ipmitool"); + case BMC_PROTOCOL__REDFISHTOOL: return(BMC_PROTOCOL__REDFISHTOOL_STR); + case BMC_PROTOCOL__IPMITOOL: return(BMC_PROTOCOL__IPMITOOL_STR); default: return("unknown"); } } @@ -157,15 +157,71 @@ int bmcUtil_init ( void ) void bmcUtil_info_init ( bmc_info_type & bmc_info ) { - bmc_info.device_id.clear(); - bmc_info.manufacturer_name.clear(); + bmc_info.manufacturer.clear(); bmc_info.manufacturer_id.clear(); bmc_info.product_name.clear(); bmc_info.product_id.clear(); + bmc_info.device_id.clear(); + bmc_info.fw_version.clear(); bmc_info.hw_version.clear(); + + bmc_info.power_on = false ; + bmc_info.restart_cause.clear() ; } +/************************************************************************* + * + * Name : bmcUtil_hwmon_info + * + * Purpose : Creates the hardware monitor info file and content. + * + * Description: The hardware monitor learns the hosts power state and + * current bmc protocol being used. + * + * Future : An extra string is passed in but currently unused. + * + * Returns : nothing + * + *************************************************************************/ + +void bmcUtil_hwmon_info ( string hostname, + bmc_protocol_enum proto, + bool power_on, + string extra ) +{ + + /* default the bmc info file */ + string bmc_info_path_n_filename = BMC_OUTPUT_DIR + hostname ; + + /* remove the old BMC info file if present */ + daemon_remove_file ( bmc_info_path_n_filename.data() ); + + /* add the 'protocol' key:val pair */ + string info_str = "{\"protocol\":\"" ; + if ( proto == BMC_PROTOCOL__REDFISHTOOL ) + info_str.append(BMC_PROTOCOL__REDFISHTOOL_STR); + else + info_str.append(BMC_PROTOCOL__IPMITOOL_STR); + + /* add the 'power' state key:val pair */ + if ( power_on ) + info_str.append("\",\"power\":\"on\""); + else + info_str.append("\",\"power\":\"off\""); + + /* add the extra data if it exists */ + if ( ! extra.empty () ) + info_str.append(extra); + + /* terminate */ + info_str.append ("}"); + + blog ("%s hwmon info: %s", hostname.c_str(), info_str.c_str()); + + /* write the data to the file */ + daemon_log ( bmc_info_path_n_filename.data(), info_str.data() ); +} /************************************************************************* * diff --git a/mtce-common/src/common/bmcUtil.h b/mtce-common/src/common/bmcUtil.h index 2faea427..47d01c5a 100644 --- a/mtce-common/src/common/bmcUtil.h +++ b/mtce-common/src/common/bmcUtil.h @@ -13,24 +13,57 @@ * Starling-X BMC Utilities Header */ +#include + +using namespace std; + #include "nodeBase.h" /* for ... */ #include "threadUtil.h" /* for ... thread_info_type and utilities */ #define BMC_OUTPUT_DIR ((const char *)("/var/run/bmc/")) -#define BMC_DEFAULT_INFO ((const char *)("{\"power_state\":\"off\",\"protocol\":\"ipmitool\"}")) +/* supported protocol strings */ +#define BMC_PROTOCOL__IPMITOOL_STR ((const char *)("ipmitool")) +#define BMC_PROTOCOL__REDFISHTOOL_STR ((const char *)("redfishtool")) /* important BMC query info to log and track */ typedef struct { + /* common */ + std::string manufacturer ; + + /* IPMI Product Info */ + /* ---------------------------*/ + /* product info */ std::string product_name ; std::string product_id ; - std::string manufacturer_name ; std::string manufacturer_id ; std::string device_id ; + std::string serial_number ; + + /* hw/fw info */ std::string fw_version ; std::string hw_version ; + + /* redfish product info */ + /* -------------------------- */ + std::string bios_ver ; + std::string bmc_ver ; + std::string pn ; + std::string mn ; + std::string sn ; + + /* actions */ + std::list allowable_reset_action_list ; + + /* state info */ + std::string restart_cause ; + bool power_on ; + unsigned int memory_in_gigs ; + unsigned int processors ; + std::list links_list ; + } bmc_info_type ; @@ -86,6 +119,12 @@ string bmcUtil_create_data_fn ( string & hostname, string file_suffix, bmc_protocol_enum protocol ); +/* this utility creates the bmc info file for hardware monitor */ +void bmcUtil_hwmon_info ( string hostname, + bmc_protocol_enum proto, + bool power_on, + string extra ); + #include "ipmiUtil.h" /* for ... mtce-common ipmi utility header */ #include "redfishUtil.h" /* for ... mtce-common redfish utility header */ diff --git a/mtce-common/src/common/ipmiUtil.cpp b/mtce-common/src/common/ipmiUtil.cpp index c9650caf..fa45fb97 100644 --- a/mtce-common/src/common/ipmiUtil.cpp +++ b/mtce-common/src/common/ipmiUtil.cpp @@ -130,7 +130,7 @@ void ipmiUtil_bmc_info_log ( string hostname, bmc_info_type & bmc_info, int rc ) { ilog ("%s Manufacturer: %s [id:%s] [ Device: %s ver %s ]\n", hostname.c_str(), - bmc_info.manufacturer_name.c_str(), + bmc_info.manufacturer.c_str(), bmc_info.manufacturer_id.c_str(), bmc_info.device_id.c_str(), bmc_info.hw_version.c_str()); @@ -229,7 +229,7 @@ int ipmiUtil_bmc_info_load ( string hostname, const char * filename, bmc_info_ty continue; if ( _got_delimited_value ( buffer, BMC_INFO_LABEL_MANUFACTURE_ID, BMC_INFO_LABEL_DELIMITER, bmc_info.manufacturer_id )) continue; - if ( _got_delimited_value ( buffer, BMC_INFO_LABEL_MANUFACTURE_NAME, BMC_INFO_LABEL_DELIMITER, bmc_info.manufacturer_name )) + if ( _got_delimited_value ( buffer, BMC_INFO_LABEL_MANUFACTURE_NAME, BMC_INFO_LABEL_DELIMITER, bmc_info.manufacturer )) continue; else blog3 ("buffer: %s\n", buffer ); diff --git a/mtce-common/src/common/redfishUtil.cpp b/mtce-common/src/common/redfishUtil.cpp index 6909592c..b94cd723 100644 --- a/mtce-common/src/common/redfishUtil.cpp +++ b/mtce-common/src/common/redfishUtil.cpp @@ -15,6 +15,7 @@ using namespace std; #include "nodeBase.h" /* for ... mtce node common definitions */ +#include "nodeUtil.h" /* for ... tolowercase */ #include "hostUtil.h" /* for ... mtce host common definitions */ #include "jsonUtil.h" /* for ... */ #include "redfishUtil.h" /* for ... this module header */ @@ -59,7 +60,7 @@ int redfishUtil_init ( void ) bool redfishUtil_is_supported (string & hostname, string & response) { - if ( response.length() > strlen(REDFISH_LABEL__FW_VERSION )) + if ( response.length() > strlen(REDFISH_LABEL__REDFISH_VERSION )) { string redfish_version = "" ; @@ -83,7 +84,7 @@ bool redfishUtil_is_supported (string & hostname, string & response) /* if no error then look for the redfish version number */ if ( jsonUtil_get_key_val ((char*)response.data(), - REDFISH_LABEL__FW_VERSION, + REDFISH_LABEL__REDFISH_VERSION, redfish_version) == PASS ) { if ( ! redfish_version.empty() ) @@ -109,7 +110,7 @@ bool redfishUtil_is_supported (string & hostname, string & response) hostname.c_str(), redfish_version.c_str(), fields, major, minor, revision ); - ilog ("%s response: %s", hostname.c_str(), response.c_str()); // ERIK: make blog ? + blog ("%s response: %s", hostname.c_str(), response.c_str()); } } else @@ -123,7 +124,7 @@ bool redfishUtil_is_supported (string & hostname, string & response) { wlog ("%s bmc redfish root query response has no '%s' label\n%s", hostname.c_str(), - REDFISH_LABEL__FW_VERSION, + REDFISH_LABEL__REDFISH_VERSION, response.c_str()); } } @@ -161,6 +162,9 @@ string redfishUtil_create_request ( string cmd, /* build the command ; starting with the redfishtool binary */ string command_request = REDFISHTOOL_PATH_AND_FILENAME ; + /* allow the BMC to redirect http to https */ + command_request.append(" -S Always"); + /* specify the bmc ip address */ command_request.append(" -r "); command_request.append(ip); @@ -182,3 +186,119 @@ string redfishUtil_create_request ( string cmd, return (command_request); } + +/************************************************************************* + * + * Name : redfishUtil_get_bmc_info + * + * Purpose : + * + * Description: + * + * Returns : PASS if succesful + * FAIL_OPERATION if unsuccessful + * + ************************************************************************/ + +int redfishUtil_get_bmc_info ( string & hostname, + string & bmc_info_filename, + bmc_info_type & bmc_info ) +{ + if ( bmc_info_filename.empty() ) + { + wlog ("%s bmc info filename empty", hostname.c_str()); + return (FAIL_NO_DATA); + } + + string json_bmc_info = daemon_read_file (bmc_info_filename.data()); + if ( json_bmc_info.empty() ) + { + wlog ("%s bmc info file empty", hostname.c_str()); + return (FAIL_STRING_EMPTY) ; + } + + struct json_object *json_obj = json_tokener_parse((char*)json_bmc_info.data()); + if ( !json_obj ) + { + wlog ("%s bmc info file empty", hostname.c_str()); + return (FAIL_JSON_PARSE) ; + + } + + bmc_info.manufacturer = jsonUtil_get_key_value_string( json_obj, REDFISH_LABEL__MANUFACTURER ); + bmc_info.sn = jsonUtil_get_key_value_string( json_obj, REDFISH_LABEL__SERIAL_NUMBER); + bmc_info.mn = jsonUtil_get_key_value_string( json_obj, REDFISH_LABEL__MODEL_NUMBER ); + bmc_info.pn = jsonUtil_get_key_value_string( json_obj, REDFISH_LABEL__PART_NUMBER ); + bmc_info.bmc_ver = jsonUtil_get_key_value_string( json_obj, REDFISH_LABEL__BMC_VERSION ); + bmc_info.bios_ver = jsonUtil_get_key_value_string( json_obj, REDFISH_LABEL__BIOS_VERSION ); + + ilog ("%s manufacturer is %s", hostname.c_str(), bmc_info.manufacturer.c_str()); + ilog ("%s model number:%s part number:%s serial number:%s", + hostname.c_str(), + bmc_info.mn.c_str(), + bmc_info.pn.c_str(), + bmc_info.sn.c_str()); + + ilog ("%s BIOS firmware version is %s", + hostname.c_str(), + bmc_info.bios_ver != NONE ? bmc_info.bios_ver.c_str() : "unavailable" ); + + ilog ("%s BMC firmware version is %s", + hostname.c_str(), + bmc_info.bmc_ver != NONE ? bmc_info.bmc_ver.c_str() : "unavailable" ); + + /* load the power state */ + string power_state = tolowercase(jsonUtil_get_key_value_string( json_obj, REDFISH_LABEL__POWER_STATE)); + if ( power_state == "on" ) + bmc_info.power_on = true ; + else + bmc_info.power_on = false ; + ilog ("%s power is %s", hostname.c_str(), power_state.c_str()); + + + /* get number of processors */ + string processors = jsonUtil_get_key_value_string( json_obj, REDFISH_LABEL__PROCESSOR ); + if ( ! processors.empty() ) + { + struct json_object *proc_obj = json_tokener_parse((char*)processors.data()); + if ( proc_obj ) + { + bmc_info.processors = jsonUtil_get_key_value_int ( proc_obj, REDFISH_LABEL__COUNT ); + ilog ("%s has %d processors", hostname.c_str(), bmc_info.processors); + json_object_put(proc_obj ); + } + else + { + slog ("%s processor obj: %s", hostname.c_str(), processors.c_str()); + } + } + else + { + slog ("%s processor count unavailable", hostname.c_str()); + } + + /* get amount of memory */ + string memory = jsonUtil_get_key_value_string( json_obj, REDFISH_LABEL__MEMORY ); + if ( ! memory.empty() ) + { + struct json_object *mem_obj = json_tokener_parse((char*)memory.data()); + if ( mem_obj ) + { + bmc_info.memory_in_gigs = jsonUtil_get_key_value_int ( mem_obj, REDFISH_LABEL__MEMORY_TOTAL ); + ilog ("%s has %d gigs of memory", hostname.c_str(), bmc_info.memory_in_gigs ); + json_object_put(mem_obj ); + } + else + { + slog ("%s memory obj: %s", hostname.c_str(), memory.c_str() ); + } + } + else + { + slog ("%s memory size unavailable", hostname.c_str()); + } + + json_object_put(json_obj ); + + return PASS ; +} diff --git a/mtce-common/src/common/redfishUtil.h b/mtce-common/src/common/redfishUtil.h index dcc0e2f6..d6492ba8 100644 --- a/mtce-common/src/common/redfishUtil.h +++ b/mtce-common/src/common/redfishUtil.h @@ -18,9 +18,39 @@ #define REDFISHTOOL_PATH_AND_FILENAME ((const char *)("/usr/bin/redfishtool")) #define REDFISHTOOL_OUTPUT_DIR ((const char *)("/var/run/bmc/redfishtool/")) -#define REDFISH_LABEL__FW_VERSION ((const char *)("RedfishVersion")) -#define REDFISH_MIN_MAJOR_VERSION (1) +/* generic labels */ +#define REDFISH_LABEL__STATUS ((const char *)("Status")) +#define REDFISH_LABEL__STATE ((const char *)("State")) +#define REDFISH_LABEL__HEALTH ((const char *)("Health")) +#define REDFISH_LABEL__COUNT ((const char *)("Count")) +#define REDFISH_LABEL__MODEL ((const char *)("Model")) +/* redfish version */ +#define REDFISH_MIN_MAJOR_VERSION (1) +#define REDFISH_LABEL__REDFISH_VERSION ((const char *)("RedfishVersion")) + +/* bmc info labels */ +#define REDFISH_LABEL__BMC_VERSION ((const char *)("BmcVersion")) +#define REDFISH_LABEL__SERIAL_NUMBER ((const char *)("SerialNumber")) +#define REDFISH_LABEL__PART_NUMBER ((const char *)("PartNumber")) +#define REDFISH_LABEL__MANUFACTURER ((const char *)("Manufacturer")) +#define REDFISH_LABEL__BIOS_VERSION ((const char *)("BiosVersion")) +#define REDFISH_LABEL__MODEL_NUMBER ((const char *)("Model")) +#define REDFISH_LABEL__POWER_STATE ((const char *)("PowerState")) + +/* server memory size labels */ +#define REDFISH_LABEL__MEMORY ((const char *)("MemorySummary")) +#define REDFISH_LABEL__MEMORY_TOTAL ((const char *)("TotalSystemMemoryGiB")) + +/* server processor info label */ +#define REDFISH_LABEL__PROCESSOR ((const char *)("ProcessorSummary")) + +/* supported actions */ +#define REDFISH_LABEL__ACTIONS ((const char *)("Actions")) +#define REDFISH_LABEL__ACTION_RESET ((const char *)("#ComputerSystem.Reset")) +#define REDFISH_LABEL__ACTION_RESET_ALLOWED ((const char *)("ResetType@Redfish.AllowableValues")) + +/* maintenance administrative action commands */ #define REDFISHTOOL_ROOT_QUERY_CMD ((const char *)("root")) #define REDFISHTOOL_BMC_INFO_CMD ((const char *)("Systems get")) #define REDFISHTOOL_POWER_RESET_CMD ((const char *)("Systems reset GracefulRestart")) @@ -50,4 +80,9 @@ string redfishUtil_create_request ( string cmd, /* interpret redfish root query response and check version */ bool redfishUtil_is_supported ( string & hostname, string & root_query_response ); +/* get, load and log bmc info from redfish info query response */ +int redfishUtil_get_bmc_info ( string & hostname, + string & response, + bmc_info_type & bmc_info ); + #endif // __INCLUDE_REDFISHUTIL_H__ diff --git a/mtce-common/src/common/threadUtil.cpp b/mtce-common/src/common/threadUtil.cpp index ff54edfd..0bfb1905 100644 --- a/mtce-common/src/common/threadUtil.cpp +++ b/mtce-common/src/common/threadUtil.cpp @@ -706,7 +706,7 @@ int thread_handler ( thread_ctrl_type & ctrl, thread_info_type & info ) { if ( info.status ) { - wlog ("%s %s thread completed (rc:%d)\n", + blog ("%s %s thread completed (rc:%d)\n", ctrl.hostname.c_str(), ctrl.name.c_str(), info.status); diff --git a/mtce-common/src/daemon/daemon_files.cpp b/mtce-common/src/daemon/daemon_files.cpp index 5d86af03..653b2ef2 100755 --- a/mtce-common/src/daemon/daemon_files.cpp +++ b/mtce-common/src/daemon/daemon_files.cpp @@ -1004,7 +1004,7 @@ string daemon_read_file ( const char * filename ) while ( fgets (buffer, BUFFER, _stream) ) { data.append(buffer); - if ( ++lines > 100 ) + if ( ++lines > 5000 ) { wlog ("%s file to big ; aborting\n", filename ); break ; diff --git a/mtce/src/common/nodeClass.cpp b/mtce/src/common/nodeClass.cpp index 51584876..d4c86fe0 100755 --- a/mtce/src/common/nodeClass.cpp +++ b/mtce/src/common/nodeClass.cpp @@ -638,7 +638,12 @@ nodeLinkClass::node* nodeLinkClass::addNode( string hostname ) ptr->bm_pw = NONE ; ptr->bmc_provisioned = false ; /* assume not provisioned until learned */ - ptr->power_on = false ; /* learned on first BMC connection */ + + if ( hostname == my_hostname ) + ptr->power_on = true ; + else + ptr->power_on = false ; /* learned on first BMC connection */ + bmc_access_data_init ( ptr ); /* init all the BMC access vars all modes */ /* init the alarm array only to have it updated later @@ -711,11 +716,11 @@ nodeLinkClass::node* nodeLinkClass::addNode( string hostname ) ptr->pulse_link[i].next_ptr = NULL ; ptr->pulse_link[i].prev_ptr = NULL ; ptr->monitor[i] = false ; - ptr->hbs_minor[i] = false ; - ptr->hbs_degrade[i] = false ; - ptr->hbs_failure[i] = false ; + ptr->hbs_minor[i] = false ; + ptr->hbs_degrade[i] = false ; + ptr->hbs_failure[i] = false ; ptr->max_count[i] = 0 ; - ptr->hbs_count[i] = 0 ; + ptr->hbs_count[i] = 0 ; ptr->hbs_minor_count[i] = 0 ; ptr->hbs_misses_count[i] = 0 ; ptr->b2b_misses_count[i] = 0 ; @@ -727,15 +732,15 @@ nodeLinkClass::node* nodeLinkClass::addNode( string hostname ) ptr->health = NODE_HEALTH_UNKNOWN ; - ptr->pmon_missing_count = 0; - ptr->pmon_degraded = false ; + ptr->pmon_missing_count = 0; + ptr->pmon_degraded = false ; /* now add it to the node list ; dealing with all conditions */ - + /* if the node list is empty add it to the head */ if( head == NULL ) { - head = ptr ; + head = ptr ; tail = ptr ; ptr->prev = NULL ; ptr->next = NULL ; @@ -749,7 +754,7 @@ nodeLinkClass::node* nodeLinkClass::addNode( string hostname ) tail->next = ptr ; ptr->prev = tail ; ptr->next = NULL ; - tail = ptr ; + tail = ptr ; } /* start with no action and an empty todo list */ @@ -803,7 +808,7 @@ struct nodeLinkClass::node* nodeLinkClass::getNode ( string hostname ) } -struct nodeLinkClass::node* nodeLinkClass::getEventBaseNode ( libEvent_enum request, +struct nodeLinkClass::node* nodeLinkClass::getEventBaseNode ( libEvent_enum request, struct event_base * base_ptr) { struct node * ptr = static_cast(NULL) ; @@ -823,7 +828,7 @@ struct nodeLinkClass::node* nodeLinkClass::getEventBaseNode ( libEvent_enum requ { if ( ptr->sysinvEvent.base == base_ptr ) { - hlog1 ("%s Found Sysinv Event Base Pointer (%p)\n", + hlog1 ("%s Found Sysinv Event Base Pointer (%p)\n", ptr->hostname.c_str(), ptr->sysinvEvent.base); return ptr ; @@ -906,7 +911,7 @@ int nodeLinkClass::remNode( string hostname ) #ifdef WANT_PULSE_LIST_SEARCH_ON_DELETE /* Splice the node out of the pulse monitor list */ - + for ( int i = 0 ; i < MAX_IFACES ; i++ ) { /* Does the pulse monitor list exist ? */ @@ -919,15 +924,15 @@ int nodeLinkClass::remNode( string hostname ) { pulse_list[i].head_ptr = NULL ; pulse_list[i].tail_ptr = NULL ; - dlog ("Pulse: Single Node -> Head Case\n"); + dlog ("Pulse: Single Node -> Head Case\n"); } else { dlog ("Pulse: Multiple Nodes -> Head Case\n"); pulse_list[i].head_ptr = pulse_list[i].head_ptr->pulse_link[i].next_ptr ; - pulse_list[i].head_ptr->pulse_link[i].prev_ptr = NULL ; + pulse_list[i].head_ptr->pulse_link[i].prev_ptr = NULL ; } - } + } else if ( pulse_list[i].tail_ptr == pulse_ptr ) { dlog ("Pulse: Multiple Node -> Tail Case\n"); @@ -961,7 +966,7 @@ int nodeLinkClass::remNode( string hostname ) { dlog ("Multiple Nodes -> Head Case\n"); head = head->next ; - head->prev = NULL ; + head->prev = NULL ; delNode ( ptr ); rc = PASS ; } @@ -4086,9 +4091,6 @@ int nodeLinkClass::set_bm_prov ( struct nodeLinkClass::node * node_ptr, bool sta int rc = FAIL_HOSTNAME_LOOKUP ; if ( node_ptr != NULL ) { - /* default the bmc info file */ - string bmc_info_path_n_filename = BMC_OUTPUT_DIR + node_ptr->hostname ; - ilog ("%s bmc %sprovision request (provisioned:%s)\n", node_ptr->hostname.c_str(), state ? "" : "de", @@ -4097,8 +4099,9 @@ int nodeLinkClass::set_bm_prov ( struct nodeLinkClass::node * node_ptr, bool sta /* Clear the alarm if we are starting fresh from an unprovisioned state */ if (( node_ptr->bmc_provisioned == false ) && ( state == true )) { - /* default the bmc info file */ - daemon_log ( bmc_info_path_n_filename.data(), BMC_DEFAULT_INFO ); + bmcUtil_hwmon_info ( node_ptr->hostname, + node_ptr->bmc_protocol, + node_ptr->power_on, "" ); ilog ("%s starting BM ping monitor to address '%s'\n", node_ptr->hostname.c_str(), @@ -4136,7 +4139,8 @@ int nodeLinkClass::set_bm_prov ( struct nodeLinkClass::node * node_ptr, bool sta /* handle the case going from provisioned to not provisioned */ else if (( node_ptr->bmc_provisioned == true ) && ( state == false )) { - /* remove the BMC info file */ + /* remove the old BMC info file if present */ + string bmc_info_path_n_filename = BMC_OUTPUT_DIR + node_ptr->hostname ; daemon_remove_file ( bmc_info_path_n_filename.data() ); ilog ("%s deprovisioning bmc ; accessible:%s\n", diff --git a/mtce/src/heartbeat/hbsStubs.cpp b/mtce/src/heartbeat/hbsStubs.cpp index 2b945186..b92db539 100644 --- a/mtce/src/heartbeat/hbsStubs.cpp +++ b/mtce/src/heartbeat/hbsStubs.cpp @@ -359,6 +359,17 @@ void nodeLinkClass::bmc_command_done ( struct nodeLinkClass::node * node_ptr ) } +void bmcUtil_hwmon_info ( string hostname, + bmc_protocol_enum proto, + bool power_on, + string extra ) +{ + UNUSED(hostname); + UNUSED(proto); + UNUSED(power_on); + UNUSED(extra); +} + int mtcAlarm_clear ( string hostname, mtc_alarm_id_enum id ) { UNUSED(hostname); id = id ; return (PASS); } int mtcAlarm_warning ( string hostname, mtc_alarm_id_enum id ) { UNUSED(hostname); id = id ; return (PASS); } int mtcAlarm_minor ( string hostname, mtc_alarm_id_enum id ) { UNUSED(hostname); id = id ; return (PASS); } diff --git a/mtce/src/maintenance/mtcBmcUtil.cpp b/mtce/src/maintenance/mtcBmcUtil.cpp index 595209c0..1f79d376 100644 --- a/mtce/src/maintenance/mtcBmcUtil.cpp +++ b/mtce/src/maintenance/mtcBmcUtil.cpp @@ -205,8 +205,6 @@ int nodeLinkClass::bmc_command_recv ( struct nodeLinkClass::node * node_ptr ) } else { - ilog ("TODO: Handle RedfishTool Response:\n%s", - node_ptr->bmc_thread_info.data.c_str() ); rc = PASS ; } } @@ -214,14 +212,19 @@ int nodeLinkClass::bmc_command_recv ( struct nodeLinkClass::node * node_ptr ) { if (( rc = node_ptr->bmc_thread_info.status ) != PASS ) { - elog ("%s %s command failed (%s) (data:%s) (rc:%d:%d:%s)\n", - node_ptr->hostname.c_str(), - bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), - bmcUtil_getProtocol_str(node_ptr->bmc_protocol).c_str(), - node_ptr->bmc_thread_info.data.c_str(), - rc, - node_ptr->bmc_thread_info.status, - node_ptr->bmc_thread_info.status_string.c_str()); + /* Don't log an error if this is just the BMC Query failure + * used for protocol learning */ + if ( node_ptr->bmc_thread_info.command != BMC_THREAD_CMD__BMC_QUERY ) + { + elog ("%s %s command failed (%s) (data:%s) (rc:%d:%d:%s)\n", + node_ptr->hostname.c_str(), + bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), + bmcUtil_getProtocol_str(node_ptr->bmc_protocol).c_str(), + node_ptr->bmc_thread_info.data.c_str(), + rc, + node_ptr->bmc_thread_info.status, + node_ptr->bmc_thread_info.status_string.c_str()); + } } else { diff --git a/mtce/src/maintenance/mtcCtrlMsg.cpp b/mtce/src/maintenance/mtcCtrlMsg.cpp index 67d9a7a9..c6a9c69d 100755 --- a/mtce/src/maintenance/mtcCtrlMsg.cpp +++ b/mtce/src/maintenance/mtcCtrlMsg.cpp @@ -246,8 +246,11 @@ int mtc_service_inbox ( nodeLinkClass * obj_ptr, obj_ptr->set_cmd_resp ( hostname , msg ) ; if ( msg.num > 0 ) { + /* log if not locked message, not start host services result + * message and there is an error */ if (( msg.cmd != MTC_MSG_LOCKED ) && - ( msg.cmd != MTC_CMD_HOST_SVCS_RESULT )) + ( msg.cmd != MTC_CMD_HOST_SVCS_RESULT ) && + ( msg.parm[0] )) { ilog ("%s '%s' ACK (rc:%d) (%s)", hostname.c_str(), diff --git a/mtce/src/maintenance/mtcNodeHdlrs.cpp b/mtce/src/maintenance/mtcNodeHdlrs.cpp index b400b94a..d2f07e40 100755 --- a/mtce/src/maintenance/mtcNodeHdlrs.cpp +++ b/mtce/src/maintenance/mtcNodeHdlrs.cpp @@ -3826,6 +3826,7 @@ int nodeLinkClass::reset_handler ( struct nodeLinkClass::node * node_ptr ) case MTC_RESET__FAIL: { elog ("%s Reset failed ; aborting after max retries\n", node_ptr->hostname.c_str()); + stop_offline_handler ( node_ptr ); mtcInvApi_update_task ( node_ptr, MTC_TASK_RESET_FAIL); mtcTimer_reset ( node_ptr->mtcTimer ); mtcTimer_start ( node_ptr->mtcTimer, mtcTimer_handler, MTC_TASK_UPDATE_DELAY ); @@ -4866,51 +4867,51 @@ int nodeLinkClass::power_handler ( struct nodeLinkClass::node * node_ptr ) break ; } - rc = bmc_command_send ( node_ptr, BMC_THREAD_CMD__POWER_STATUS ) ; - if ( rc ) - { - node_ptr->power_action_retries-- ; - powerStageChange ( node_ptr , MTC_POWERON__QUEUE ); - } - else - { - powerStageChange ( node_ptr , MTC_POWERON__POWER_STATUS_WAIT ); - } - mtcTimer_reset ( node_ptr->mtcTimer ); - mtcTimer_start ( node_ptr->mtcTimer, mtcTimer_handler, MTC_IPMITOOL_REQUEST_DELAY ); - break ; + rc = bmc_command_send ( node_ptr, BMC_THREAD_CMD__POWER_STATUS ) ; + if ( rc ) + { + node_ptr->power_action_retries-- ; + powerStageChange ( node_ptr , MTC_POWERON__QUEUE ); } + else + { + powerStageChange ( node_ptr , MTC_POWERON__POWER_STATUS_WAIT ); + } + mtcTimer_reset ( node_ptr->mtcTimer ); + mtcTimer_start ( node_ptr->mtcTimer, mtcTimer_handler, MTC_IPMITOOL_REQUEST_DELAY ); + break ; + } case MTC_POWERON__POWER_STATUS_WAIT: { - if ( mtcTimer_expired ( node_ptr->mtcTimer ) ) + if ( mtcTimer_expired ( node_ptr->mtcTimer ) ) + { + rc = bmc_command_recv ( node_ptr ); + if ( rc == RETRY ) { - rc = bmc_command_recv ( node_ptr ); - if ( rc == RETRY ) + mtcTimer_start ( node_ptr->mtcTimer, mtcTimer_handler, MTC_RETRY_WAIT ); + } + else if ( rc == PASS ) + { + if ( node_ptr->bmc_thread_info.data.find (IPMITOOL_POWER_ON_STATUS) != std::string::npos ) { - mtcTimer_start ( node_ptr->mtcTimer, mtcTimer_handler, MTC_RETRY_WAIT ); - } - else if ( rc == PASS ) - { - if ( node_ptr->bmc_thread_info.data.find (IPMITOOL_POWER_ON_STATUS) != std::string::npos ) - { - ilog ("%s power is already on ; no action required\n", node_ptr->hostname.c_str()); - node_ptr->power_on = true ; - mtcInvApi_update_task ( node_ptr, "Power Already On" ); - mtcTimer_start ( node_ptr->mtcTimer, mtcTimer_handler, MTC_TASK_UPDATE_DELAY ); - powerStageChange ( node_ptr , MTC_POWERON__DONE ); - } - else - { - node_ptr->power_on = false ; - ilog ("%s power is off ; powering on ...\n", node_ptr->hostname.c_str() ); - powerStageChange ( node_ptr , MTC_POWERON__REQ_SEND ); - } + ilog ("%s power is already on ; no action required\n", node_ptr->hostname.c_str()); + node_ptr->power_on = true ; + mtcInvApi_update_task ( node_ptr, "Power Already On" ); + mtcTimer_start ( node_ptr->mtcTimer, mtcTimer_handler, MTC_TASK_UPDATE_DELAY ); + powerStageChange ( node_ptr , MTC_POWERON__DONE ); } else { - powerStageChange ( node_ptr , MTC_POWERON__POWER_STATUS ); + node_ptr->power_on = false ; + ilog ("%s power is off ; powering on ...\n", node_ptr->hostname.c_str() ); + powerStageChange ( node_ptr , MTC_POWERON__REQ_SEND ); } } + else + { + powerStageChange ( node_ptr , MTC_POWERON__POWER_STATUS ); + } + } break ; } case MTC_POWERON__REQ_SEND: @@ -5047,6 +5048,12 @@ int nodeLinkClass::power_handler ( struct nodeLinkClass::node * node_ptr ) ar_enable ( node_ptr ); + /* tell the hardware monitor of the power state and protocol */ + bmcUtil_hwmon_info ( node_ptr->hostname, + node_ptr->bmc_protocol, + node_ptr->power_on, "" ); + send_hwmon_command ( node_ptr->hostname, MTC_CMD_MOD_HOST ); + mtcInvApi_force_task ( node_ptr, "" ); break ; } @@ -5656,6 +5663,12 @@ int nodeLinkClass::powercycle_handler ( struct nodeLinkClass::node * node_ptr ) recoveryStageChange ( node_ptr, MTC_RECOVERY__START); /* reset the fsm */ disableStageChange ( node_ptr, MTC_DISABLE__START); /* reset the fsm */ + /* tell the hardware monitor of the power state and protocol */ + bmcUtil_hwmon_info ( node_ptr->hostname, + node_ptr->bmc_protocol, + node_ptr->power_on, "" ); + send_hwmon_command ( node_ptr->hostname, MTC_CMD_MOD_HOST ); + plog ("%s Power-Cycle Completed (uptime:%d)\n", node_ptr->hostname.c_str(), node_ptr->uptime ); } break ; @@ -6258,13 +6271,22 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr ) } else if ( rc != PASS ) { - wlog ("%s %s recv failed (rc:%d:%d:%s); defaulting to ipmi", - node_ptr->hostname.c_str(), - bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), - rc, - node_ptr->bmc_thread_info.status, - node_ptr->bmc_thread_info.status_string.c_str()); - + if (( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__BMC_QUERY ) && + ( rc == FAIL_SYSTEM_CALL )) + { + wlog ("%s bmc does not support Redfish ; " \ + "defaulting to ipmi", + node_ptr->hostname.c_str()); + } + else + { + wlog ("%s %s recv failed (rc:%d:%d:%s); defaulting to ipmi", + node_ptr->hostname.c_str(), + bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), + rc, + node_ptr->bmc_thread_info.status, + node_ptr->bmc_thread_info.status_string.c_str()); + } node_ptr->bmc_protocol_learning = false ; node_ptr->bmc_protocol_learned = true ; node_ptr->bmc_protocol = BMC_PROTOCOL__IPMITOOL ; @@ -6314,14 +6336,103 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr ) ( node_ptr->bmc_info_query_done == false ) && ( mtcTimer_expired (node_ptr->bm_timer ) == true )) { - ilog ("%s bmc redfish info query ; forthcoming", - node_ptr->hostname.c_str()); + if (( node_ptr->bmc_info_query_active == false ) && + ( node_ptr->bmc_info_query_done == false )) + { + if ( bmc_command_send ( node_ptr, BMC_THREAD_CMD__BMC_INFO ) != PASS ) + { + elog ("%s bmc redfish '%s' send failed\n", + node_ptr->hostname.c_str(), + bmcUtil_getCmd_str( + node_ptr->bmc_thread_info.command).c_str()); + node_ptr->bmc_protocol = BMC_PROTOCOL__IPMITOOL ; + } + else + { + node_ptr->bmc_info_query_active = true ; + ilog ("%s bmc redfish '%s' in progress", /* ERIK: blog */ + node_ptr->hostname.c_str(), + bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str()); + mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_FIRST_WAIT ); + } + } + else if (( node_ptr->bmc_info_query_active == true ) && + ( node_ptr->bmc_info_query_done == false)) + { + int rc ; + if ( ( rc = bmc_command_recv ( node_ptr ) ) == RETRY ) + { + mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_RETRY_WAIT ); + } + else if ( rc != PASS ) + { + /* this error is reported by the receive driver */ + node_ptr->bmc_info_query_active = false ; + node_ptr->bmc_thread_ctrl.done = true ; + } + else + { + node_ptr->bmc_thread_ctrl.done = true ; - /* Will implement query info in next update */ - // node_ptr->bmc_info_query_done = true ; + /* get the bmc info string the read thread provided */ + if ( redfishUtil_get_bmc_info ( + node_ptr->hostname, + node_ptr->bmc_thread_info.data, + node_ptr->bmc_info ) != PASS ) + { + elog ("%s bmc %s failed ; defaulting to ipmitool", + node_ptr->hostname.c_str(), + bmcUtil_getCmd_str( + node_ptr->bmc_thread_info.command).c_str()); - /* TEMP: reverting back to ipmi. */ - node_ptr->bmc_protocol = BMC_PROTOCOL__IPMITOOL ; + node_ptr->bmc_info_query_active = false ; + node_ptr->bmc_info_query_done = false ; + node_ptr->bmc_protocol = BMC_PROTOCOL__IPMITOOL ; + } + else + { + mtcTimer_reset ( node_ptr->bm_timer ); + +#ifdef REDFISH_INTEGRATION_DONE + /* success path */ + node_ptr->bmc_info_query_active = false ; + node_ptr->bmc_info_query_done = true ; + node_ptr->bmc_protocol = BMC_PROTOCOL__REDFISHTOOL ; + + mtcTimer_reset ( node_ptr->bmc_access_timer ); + node_ptr->bmc_accessible = true ; + + /* save the host's power state */ + node_ptr->power_on = node_ptr->bmc_info.power_on ; + + plog ("%s bmc is accessible using redfish", + node_ptr->hostname.c_str()); + + + /* tell the hardware monitor of the power state and protocol */ + bmcUtil_hwmon_info ( node_ptr->hostname, + node_ptr->bmc_protocol, + node_ptr->power_on, "" ); + + send_hwmon_command ( node_ptr->hostname, MTC_CMD_MOD_HOST ); +#else + /* Redfish Power Control Commands not Implemented Yet + * Redfish not fully integrated. + * Need to continue to default to IPMI + * + * Start */ + node_ptr->bmc_accessible = false ; + node_ptr->bmc_info_query_active = false ; + node_ptr->bmc_info_query_done = false ; + node_ptr->bmc_protocol = BMC_PROTOCOL__IPMITOOL ; + /* End */ +#endif + + node_ptr->bmc_thread_ctrl.done = true ; + node_ptr->bmc_thread_info.command = 0 ; + } + } + } } /***************************************************************** @@ -6505,7 +6616,10 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr ) wlog ("%s is powered off while in the unlocked state\n", node_ptr->hostname.c_str()); availStatusChange ( node_ptr, MTC_AVAIL_STATUS__POWERED_OFF ); } - } /* end power off deteeeeeection handling */ + } /* end power off detection handling */ + + bmcUtil_hwmon_info ( node_ptr->hostname, node_ptr->bmc_protocol, node_ptr->power_on, "" ); + } /* end query handling success path */ } /* end power status query handling */ } /* end query info stages handling */ diff --git a/mtce/src/maintenance/mtcThreads.cpp b/mtce/src/maintenance/mtcThreads.cpp index e5113f97..9e38f67f 100644 --- a/mtce/src/maintenance/mtcThreads.cpp +++ b/mtce/src/maintenance/mtcThreads.cpp @@ -241,13 +241,17 @@ void * mtcThread_bmc ( void * arg ) dlog_t ("%s '%s' command\n", info_ptr->log_prefix, command.c_str()); /*************** create the password file ***************/ - /* password file contains username and password in format + /* password file contains user name and password in format * * {"username":"","password":""} * + * FIXME: Need to settle on username or user. + * Support both for now. */ string config_file_content = "{\"username\":\"" ; config_file_content.append(extra_ptr->bm_un); + config_file_content.append("\",\"user\":\""); + config_file_content.append(extra_ptr->bm_un); config_file_content.append("\",\"password\":\""); config_file_content.append(extra_ptr->bm_pw); config_file_content.append("\"}"); @@ -324,7 +328,8 @@ void * mtcThread_bmc ( void * arg ) info_ptr->status_string = daemon_read_file(datafile.data()); } } - nodeUtil_latency_log ( info_ptr->hostname, "redfishtool system call", 1000 ); + /* produce latency log if command takes longer than 5 seconds */ + nodeUtil_latency_log ( info_ptr->hostname, "redfishtool system call", 5000 ); } #ifdef WANT_FIT_TESTING