From 3f4c2cbb45ca652d1d6876c126f6699148befdbc Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Wed, 12 Oct 2022 21:48:26 +0000 Subject: [PATCH] Mtce: Add ActionInfo extension support for reset operations. StarlingX Maintenance supports host power and reset control through both IPMI and Redfish Platform Management protocols when the host's BMC (Board Management Controller) is provisioned. The power and reset action commands for Redfish are learned through HTTP payload annotations at the Systems level; "/redfish/v1/Systems. The existing maintenance implementation only supports the "ResetType@Redfish.AllowableValues" payload property annotation at the #ComputerSystem.Reset Actions property level. However, the Redfish schema also supports an 'ActionInfo' extension at /redfish/v1/Systems/1/ResetActionInfo. This update adds support for the 'ActionInfo' extension for Reset and power control command learning. For more information refer to the section 6.3 ActionInfo 1.3.0 of the Redfish Data Model Specification link in the launchpad report. Test Plan: PASS: Verify CentOS build and patch install. PASS: Verify Debian build and ISO install. PASS: Verify with Debian redfishtool 1.1.0 and 1.5.0 PASS: Verify reset/power control cmd load from newly added second level query from ActionInfo service. Failure Handling: Significant failure path testing with this update PASS: Verify Redfish protocol is periodically retried from start when bm_type=redfish fails to connect. PASS: Verify BMC access protocol defaults to IPMI when bm_type=dynamic but failed connect using redfish. Connection failures in the above cases include - redfish bmc root query fails - redfish bmc info query fails - redfish bmc load power/reset control actions fails - missing second level Parameters label list - missing second level AllowableValues label list PASS: Verify sensor monitoring is relearned to ipmi from failed and retried with bm_type=redfish after switch to bm_type=dynamic or bm_type=ipmi by sysinv update command. Regression: PASS: Verify with CentOS redfishtool 1.1.0 PASS: Verify switch back and forth between ipmi and redfish using update bm_type=ipmi and bm_type=redfish commands PASS: Verify switch from ipmi to redfish usinf bm_type=dynamic for hosts that support redfish PASS: Verify redfish protocol is preferred in bm_type=dynamic mode PASS: Verify IPMI sensor monitoring when bm_type=ipmi PASS: Verify IPMI sensor monitoring when bm_type=dynamic and redfish connect fails. PASS: Verify redfish sensor event assert/clear handling with alarm and degrade condition for both IPMI and redfish. PASS: Verify reset/power command learn by single level query. PASS: Verify mtcAgent.log logging Closes-Bug: 1992286 Signed-off-by: Eric MacDonald Change-Id: Ie8cdbd18104008ca46fc6edf6f215e73adc3bb35 --- mtce-common/src/common/bmcUtil.cpp | 3 + mtce-common/src/common/bmcUtil.h | 4 + mtce-common/src/common/jsonUtil.cpp | 2 +- mtce-common/src/common/redfishUtil.cpp | 85 +++++-- mtce-common/src/common/redfishUtil.h | 104 +++++++-- mtce/src/common/nodeClass.cpp | 41 +++- mtce/src/common/nodeClass.h | 6 + mtce/src/maintenance/mtcBmcUtil.cpp | 37 ++- mtce/src/maintenance/mtcNodeHdlrs.cpp | 303 +++++++++++++++++++------ mtce/src/maintenance/mtcThreads.cpp | 6 + 10 files changed, 469 insertions(+), 122 deletions(-) diff --git a/mtce-common/src/common/bmcUtil.cpp b/mtce-common/src/common/bmcUtil.cpp index 964e4eca..efbb3178 100644 --- a/mtce-common/src/common/bmcUtil.cpp +++ b/mtce-common/src/common/bmcUtil.cpp @@ -154,6 +154,7 @@ int bmcUtil_init ( void ) bmcUtil_request_str_array[BMC_THREAD_CMD__POWER_STATUS] = "Query Power Status"; bmcUtil_request_str_array[BMC_THREAD_CMD__RESTART_CAUSE] = "Query Reset Reason"; bmcUtil_request_str_array[BMC_THREAD_CMD__BOOTDEV_PXE] = "Netboot"; + bmcUtil_request_str_array[BMC_THREAD_CMD__RAW_GET] = "Raw Get"; bmcUtil_request_str_array[BMC_THREAD_CMD__READ_SENSORS] = "Read Sensors"; bmcUtil_request_str_array[BMC_THREAD_CMD__LAST] = "unknown"; @@ -203,6 +204,8 @@ void bmcUtil_info_init ( bmc_info_type & bmc_info ) bmc_info.power_ctrl.poweron.immediate = "" ; bmc_info.power_ctrl.poweroff.graceful = "" ; bmc_info.power_ctrl.poweroff.immediate = "" ; + + bmc_info.power_ctrl.raw_target_path = "" ; } /************************************************************************* diff --git a/mtce-common/src/common/bmcUtil.h b/mtce-common/src/common/bmcUtil.h index 8c2a351d..8f295d30 100644 --- a/mtce-common/src/common/bmcUtil.h +++ b/mtce-common/src/common/bmcUtil.h @@ -38,6 +38,8 @@ typedef struct typedef struct { + /* Set to next level url to query if required */ + string raw_target_path ; bmc_power_ctrl_type reset ; bmc_power_ctrl_type poweron ; bmc_power_ctrl_type poweroff ; @@ -101,6 +103,7 @@ typedef enum BMC_THREAD_CMD__POWER_OFF, BMC_THREAD_CMD__POWER_CYCLE, + BMC_THREAD_CMD__RAW_GET, BMC_THREAD_CMD__BMC_INFO, BMC_THREAD_CMD__POWER_STATUS, BMC_THREAD_CMD__RESTART_CAUSE, @@ -114,6 +117,7 @@ typedef enum #define BMC_QUERY_FILE_SUFFIX ((const char *)("_root_query")) #define BMC_INFO_FILE_SUFFIX ((const char *)("_bmc_info")) +#define BMC_RAW_GET_FILE_SUFFIX ((const char *)("_raw_get")) #define BMC_POWER_CMD_FILE_SUFFIX ((const char *)("_power_cmd_result")) #define BMC_RESET_CMD_FILE_SUFFIX ((const char *)("_reset")) #define BMC_BOOTDEV_CMD_FILE_SUFFIX ((const char *)("_bootdev")) diff --git a/mtce-common/src/common/jsonUtil.cpp b/mtce-common/src/common/jsonUtil.cpp index 76297d7a..67292f32 100644 --- a/mtce-common/src/common/jsonUtil.cpp +++ b/mtce-common/src/common/jsonUtil.cpp @@ -108,7 +108,7 @@ static struct json_object * _json_verify_object ( struct json_object * obj, { return (req_obj); } - wlog ("Specified label '%s' not found in response\n", label ); + jlog ("Specified label '%s' not found in response\n", label ); status = json_object_object_get_ex (obj, "error", &req_obj ); if (( status == true ) && ( req_obj )) { diff --git a/mtce-common/src/common/redfishUtil.cpp b/mtce-common/src/common/redfishUtil.cpp index 5bdfd758..a8d09d02 100644 --- a/mtce-common/src/common/redfishUtil.cpp +++ b/mtce-common/src/common/redfishUtil.cpp @@ -78,7 +78,7 @@ int redfishUtil_init ( void ) /************************************************************************* * - * Name : _load_action_lists + * Name : redfishUtil_load_actions * * Purpose : Load supported host actions. * @@ -92,9 +92,9 @@ int redfishUtil_init ( void ) * *************************************************************************/ -void _load_action_lists ( string & hostname, - bmc_info_type & bmc_info, - std::list & host_action_list) +void redfishUtil_load_actions ( string & hostname, + bmc_info_type & bmc_info, + std::list & host_action_list) { /* Walk through the host action list looking for and updating * this host's bmc_info supported actions lists */ @@ -131,7 +131,7 @@ void _load_action_lists ( string & hostname, if (( bmc_info.power_ctrl.reset.graceful.empty() ) && ( bmc_info.power_ctrl.reset.immediate.empty() )) { - elog("%s bmc offers no 'Reset' commands (%s:%s)", + wlog("%s bmc offers no 'Reset' commands (%s:%s)", hostname.c_str(), REDFISHTOOL_RESET__GRACEFUL_RESTART, REDFISHTOOL_RESET__FORCE_RESTART); @@ -156,7 +156,7 @@ void _load_action_lists ( string & hostname, if (( bmc_info.power_ctrl.poweron.graceful.empty() ) && ( bmc_info.power_ctrl.poweron.immediate.empty() )) { - elog("%s bmc offers no 'Power-On' commands (%s:%s)", + wlog("%s bmc offers no 'Power-On' commands (%s:%s)", hostname.c_str(), REDFISHTOOL_POWER_ON__ON, REDFISHTOOL_POWER_ON__FORCE_ON); @@ -181,7 +181,7 @@ void _load_action_lists ( string & hostname, if (( bmc_info.power_ctrl.poweroff.graceful.empty() ) && ( bmc_info.power_ctrl.poweroff.immediate.empty() )) { - elog("%s bmc offers no 'Power-Off' commands (%s:%s)", + wlog("%s bmc offers no 'Power-Off' commands (%s:%s)", hostname.c_str(), REDFISHTOOL_POWER_OFF__GRACEFUL_SHUTDOWN, REDFISHTOOL_POWER_OFF__FORCE_OFF); @@ -548,22 +548,77 @@ int redfishUtil_get_bmc_info ( string & hostname, std::list action_list ; /* get the first level reset action label content */ - string json_actions = - jsonUtil_get_key_value_string (json_obj_actions, - REDFISHTOOL_RESET_ACTIONS_LABEL); - - if ( jsonUtil_get_list ((char*)json_actions.data(), REDFISHTOOL_RESET_ACTIONS_ALLOWED_LABEL, action_list ) == PASS ) + string json_actions = jsonUtil_get_key_value_string (json_obj_actions, + REDFISH_LABEL__ACTION_RESET); + if ( !json_actions.empty() && json_actions.compare("none") ) { - _load_action_lists ( hostname, bmc_info, action_list); + if ( jsonUtil_get_list ((char*)json_actions.data(), + REDFISH_LABEL__ACTION_RESET_ALLOWED, + action_list ) == PASS ) + { + redfishUtil_load_actions ( hostname, bmc_info, action_list); + } + else + { + /************************************************************ + * If the REDFISH_LABEL__ACTION_RESET does not contain + * the REDFISH_LABEL__ACTION_RESET_ALLOWED then tell the + * bmc_handler FSM to query the action list through the + * REDFISH_LABEL__ACTION_INFO key value target saved to + * the bmc_info.power_ctrl.raw_target_path. + ************************************************************/ + blog ("%s bmc not offering action list through %s", + hostname.c_str(), + REDFISH_LABEL__ACTION_RESET_ALLOWED ); + + struct json_object *json_actions_obj = + json_tokener_parse((char*)json_actions.data()); + if ( json_actions_obj ) + { + string json_actions_target = + jsonUtil_get_key_value_string(json_actions_obj, + REDFISH_LABEL__ACTION_INFO); + if ( !json_actions_target.empty() && json_actions_target.compare("none") ) + { + blog ("%s posting %s target %s to bmc handler", + hostname.c_str(), + REDFISH_LABEL__ACTION_INFO, + json_actions_target.c_str()); + + /* Post the target to the bmc handler to query + * using redfishtool raw GET mode */ + bmc_info.power_ctrl.raw_target_path = json_actions_target ; + } + else + { + wlog ("%s failed to get %s target", + hostname.c_str(), + REDFISH_LABEL__ACTION_INFO); + return ( FAIL_STRING_EMPTY ); + } + } + else + { + wlog ("%s null json object from %s using label %s", + hostname.c_str(), + json_actions.c_str(), + REDFISH_LABEL__ACTION_RESET_ALLOWED); + return ( FAIL_JSON_PARSE ); + } + } } else { - elog ("%s actions list get failed ; [%s]", hostname.c_str(), json_actions.c_str()); + wlog ("%s failed parse string from %s object", + hostname.c_str(), + REDFISH_LABEL__ACTION_RESET ); + return ( FAIL_JSON_PARSE ); } } else { - elog ("%s action object get failed", hostname.c_str()); + wlog ("%s action object get failed", hostname.c_str()); + return ( FAIL_JSON_PARSE ); } /* get number of processors */ diff --git a/mtce-common/src/common/redfishUtil.h b/mtce-common/src/common/redfishUtil.h index d50e4d08..c9b6d283 100644 --- a/mtce-common/src/common/redfishUtil.h +++ b/mtce-common/src/common/redfishUtil.h @@ -46,6 +46,16 @@ typedef struct } redfish_entity_status ; +/* + * + * Redfish Data Model Specification: + * https://www.dmtf.org/sites/default/files/standards/documents/DSP0268_2022.2.pdf + * + * Redfish Resource and Schema Guide: + * https://www.dmtf.org/sites/default/files/standards/documents/DSP2046_2022.2.pdf + * + */ + /* Redfish version format is #.#.# or major.minor.revision * This feature does not care about revision. * The following are the minimum version numbers for major and minor @@ -76,25 +86,14 @@ typedef struct #define REDFISHTOOL_BMC_INFO_CMD ((const char *)("Systems get")) -/* supported actions */ -#define REDFISH_LABEL__ACTIONS ((const char *)("Actions")) -#define REDFISH_LABEL__ACTION_RESET ((const char *)("#ComputerSystem.Reset")) -#define REDFISH_LABEL__ACTION_RESET_ALLOWED ((const char *)("ResetType@Redfish.AllowableValues")) +/*********** supported reset/power command actions queries ************** -/* Redfish Reset Types: - * - * https://www.dmtf.org/sites/default/files/standards/documents/DSP0268_2019.1a.pdf */ +Method 1: Single level query through root query 'Actions' -#define REDFISHTOOL_POWER_RESET_CMD ((const char *)("Systems reset ")) + #ComputerSystem.Reset + ResetType@Redfish.AllowableValues [list] -typedef enum -{ - REDFISH_ACTION__RESET, - REDFISH_ACTION__POWER_ON, - REDFISH_ACTION__POWER_OFF, -} redfish_action_enum ; - -/* Reset actions allows json block +Method 1: payload response ( level 1 ) "Actions": { "#ComputerSystem.Reset": { @@ -109,10 +108,70 @@ typedef enum } }, -*/ -#define REDFISHTOOL_RESET_ACTIONS_LABEL ((const char *)("#ComputerSystem.Reset")) /* level 1 label */ -#define REDFISHTOOL_RESET_ACTIONS_ALLOWED_LABEL ((const char *)("ResetType@Redfish.AllowableValues")) /* level 2 label */ + -----or----- +Method 2: Double level query through root query 'Actions' + + #ComputerSystem.Reset + @Redfish.ActionInfo -> /redfish/v1/Systems/1/ResetActionInfo + arameters + AllowableValues [list] + +Method 2 level 1 payload response + + "Actions": { + "#ComputerSystem.Reset": { + "target": "/redfish/v1/Systems/1/Actions/ComputerSystem.Reset", + "@Redfish.ActionInfo": "/redfish/v1/Systems/1/ResetActionInfo" + } + }, + +Method 2 level 2 payload response through @Redfish.ActionInfo target + + GET: /redfish/v1/Systems/1/ResetActionInfo + + "@odata.type": "#ActionInfo.v1_1_2.ActionInfo", + "Name": "Reset Action Info", + "Parameters": [ + { + "DataType": "String", + "AllowableValues": [ + "On", + "ForceOff", + "GracefulShutdown", + "GracefulRestart", + "ForceRestart", + "Nmi", + "ForceOn" + ], + "Required": true, + "Name": "ResetType" + } + ], + "@odata.id": "/redfish/v1/Systems/1/ResetActionInfo", + "Oem": {}, + "Id": "ResetActionInfo" +} */ + +/* First level reset/power control GET labels */ +#define REDFISH_LABEL__ACTIONS ((const char *)("Actions")) /* level 0 */ +#define REDFISH_LABEL__ACTION_RESET ((const char *)("#ComputerSystem.Reset")) /* level 1 */ +#define REDFISH_LABEL__ACTION_RESET_ALLOWED ((const char *)("ResetType@Redfish.AllowableValues")) /* level 2 */ + +/* Second level reset/power control GET labels */ +#define REDFISH_LABEL__ACTION_INFO ((const char *)("@Redfish.ActionInfo")) +#define REDFISH_LABEL__PARAMETERS ((const char *)("Parameters")) +#define REDFISH_LABEL__ALLOWABLE_VALUES ((const char *)("AllowableValues")) + +#define REDFISHTOOL_RAW_GET_CMD ((const char *)("raw GET ")) +#define REDFISHTOOL_POWER_RESET_CMD ((const char *)("Systems reset ")) + +typedef enum +{ + REDFISH_ACTION__RESET, + REDFISH_ACTION__POWER_ON, + REDFISH_ACTION__POWER_OFF, +} redfish_action_enum ; /* Reset sub-commands */ #define REDFISHTOOL_RESET__GRACEFUL_RESTART ((const char *)("GracefulRestart")) /* Perform a graceful shutdown followed by a restart of the system. */ @@ -164,5 +223,10 @@ int redfishUtil_get_bmc_info ( string & hostname, bmc_info_type & bmc_info ); string redfishUtil_get_cmd_option ( redfish_action_enum action, - std::list host_action_list ); + std::list host_action_list ); + +void redfishUtil_load_actions ( string & hostname, + bmc_info_type & bmc_info, + std::list & host_action_list); + #endif // __INCLUDE_REDFISHUTIL_H__ diff --git a/mtce/src/common/nodeClass.cpp b/mtce/src/common/nodeClass.cpp index 90dc3c68..cdccb360 100755 --- a/mtce/src/common/nodeClass.cpp +++ b/mtce/src/common/nodeClass.cpp @@ -4414,6 +4414,38 @@ void nodeLinkClass::bmc_load_protocol ( struct nodeLinkClass::node * node_ptr ) mtcInvApi_update_mtcInfo ( node_ptr ); } +/* Default the bmc handler query control variables */ +void nodeLinkClass::bmc_default_query_controls ( struct nodeLinkClass::node * node_ptr ) +{ + node_ptr->bmc_info_query_active = false ; + node_ptr->bmc_info_query_done = false ; + + node_ptr->reset_cause_query_active = false ; + node_ptr->reset_cause_query_done = false ; + + node_ptr->power_status_query_active = false ; + node_ptr->power_status_query_done = false ; + + node_ptr->bmc_actions_query_active = false ; + node_ptr->bmc_actions_query_done = false ; +} + +/* Force bmc access protocol to IPMI */ +int nodeLinkClass::bmc_default_to_ipmi ( struct nodeLinkClass::node * node_ptr ) +{ + ilog ("%s defaulting to ipmi", node_ptr->hostname.c_str()); + + /* reset all control variables */ + bmc_default_query_controls ( node_ptr ); + + node_ptr->bmc_protocol_learning = false ; + node_ptr->bmc_protocol = BMC_PROTOCOL__IPMITOOL ; + mtcInfo_set ( node_ptr, MTCE_INFO_KEY__BMC_PROTOCOL, BMC_PROTOCOL__IPMI_STR ); + mtcInvApi_update_mtcInfo ( node_ptr ); + return (PASS); +} + + void nodeLinkClass::bmc_access_data_init ( struct nodeLinkClass::node * node_ptr ) { if ( node_ptr ) @@ -4421,12 +4453,9 @@ void nodeLinkClass::bmc_access_data_init ( struct nodeLinkClass::node * node_ptr node_ptr->bm_pw.clear(); node_ptr->bmc_accessible = false ; node_ptr->bm_ping_info.ok = false ; - node_ptr->bmc_info_query_active = false ; - node_ptr->bmc_info_query_done = false ; - node_ptr->reset_cause_query_active = false ; - node_ptr->reset_cause_query_done = false ; - node_ptr->power_status_query_active = false ; - node_ptr->power_status_query_done = false ; + + /* default bmc handler query controlo variables */ + bmc_default_query_controls ( node_ptr ); node_ptr->bm_ping_info.stage = PINGUTIL_MONITOR_STAGE__OPEN ; mtcTimer_reset ( node_ptr->bm_ping_info.timer ); diff --git a/mtce/src/common/nodeClass.h b/mtce/src/common/nodeClass.h index 5df2ce56..e69224e7 100755 --- a/mtce/src/common/nodeClass.h +++ b/mtce/src/common/nodeClass.h @@ -689,6 +689,10 @@ private: bool power_status_query_active ; bool power_status_query_done ; + + bool bmc_actions_query_active ; + bool bmc_actions_query_done ; + bool power_on = false ; /* a timer used in the bmc_handler to query @@ -1160,6 +1164,8 @@ private: int set_bm_prov ( struct nodeLinkClass::node * node_ptr, bool state ); void bmc_load_protocol ( struct nodeLinkClass::node * node_ptr ); + void bmc_default_query_controls ( struct nodeLinkClass::node * node_ptr ); + int bmc_default_to_ipmi ( struct nodeLinkClass::node * node_ptr ); void set_uptime ( struct nodeLinkClass::node * node_ptr, unsigned int uptime, bool force ); diff --git a/mtce/src/maintenance/mtcBmcUtil.cpp b/mtce/src/maintenance/mtcBmcUtil.cpp index 817db71b..75d723f6 100644 --- a/mtce/src/maintenance/mtcBmcUtil.cpp +++ b/mtce/src/maintenance/mtcBmcUtil.cpp @@ -76,11 +76,28 @@ int nodeLinkClass::bmc_command_send ( struct nodeLinkClass::node * node_ptr, if ( node_ptr->bmc_thread_info.proto == BMC_PROTOCOL__REDFISHTOOL ) { - node_ptr->bm_cmd = REDFISHTOOL_POWER_RESET_CMD ; + /* set the command specific redfishtool base command string */ + if ( command == BMC_THREAD_CMD__RAW_GET ) + node_ptr->bm_cmd = REDFISHTOOL_RAW_GET_CMD ; + else + node_ptr->bm_cmd = REDFISHTOOL_POWER_RESET_CMD ; - /* build the reset/power control command */ + /* append to the reset/power control or raw get command string */ switch (command) { + case BMC_THREAD_CMD__RAW_GET: + { + if ( ! node_ptr->bmc_info.power_ctrl.raw_target_path.empty() ) + { + node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.raw_target_path); + } + else + { + elog("%s is missing the raw get target", node_ptr->hostname.c_str()); + return(FAIL_NOT_SUPPORTED); + } + break ; + } case BMC_THREAD_CMD__POWER_RESET: { /* use immediate for all retries if server supports an immediate command */ @@ -457,13 +474,15 @@ bmc_command_recv_cleanup: if ( rc != RETRY ) { - ilog ("%s %s recv '%s' command (%s) (rc:%d)", - node_ptr->hostname.c_str(), - node_ptr->bmc_thread_ctrl.name.c_str(), - bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), - bmcUtil_getProtocol_str(node_ptr->bmc_protocol).c_str(), - rc); - + if ( rc != PASS ) + { + ilog ("%s %s recv '%s' command (%s) (rc:%d)", + node_ptr->hostname.c_str(), + node_ptr->bmc_thread_ctrl.name.c_str(), + bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), + bmcUtil_getProtocol_str(node_ptr->bmc_protocol).c_str(), + rc); + } node_ptr->bmc_thread_ctrl.done = true ; node_ptr->bmc_thread_ctrl.retries = 0 ; node_ptr->bmc_thread_ctrl.id = 0 ; diff --git a/mtce/src/maintenance/mtcNodeHdlrs.cpp b/mtce/src/maintenance/mtcNodeHdlrs.cpp index 3c8d71f5..dfe9172d 100755 --- a/mtce/src/maintenance/mtcNodeHdlrs.cpp +++ b/mtce/src/maintenance/mtcNodeHdlrs.cpp @@ -6578,13 +6578,10 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr ) /* send the BMC Query request ; redfish 'root' request */ if ( bmc_command_send ( node_ptr, BMC_THREAD_CMD__BMC_QUERY ) != PASS ) { - wlog ("%s %s send failed ; defaulting to ipmi", + wlog ("%s %s send failed", node_ptr->hostname.c_str(), bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str()); - node_ptr->bmc_protocol_learning = false ; - node_ptr->bmc_protocol = BMC_PROTOCOL__IPMITOOL ; - mtcInfo_set ( node_ptr, MTCE_INFO_KEY__BMC_PROTOCOL, BMC_PROTOCOL__IPMI_STR ); - mtcInvApi_update_mtcInfo ( node_ptr ); + return ( bmc_default_to_ipmi ( node_ptr ) ); } else { @@ -6613,26 +6610,21 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr ) if (( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__BMC_QUERY ) && (( rc == FAIL_SYSTEM_CALL ) || ( rc == FAIL_NOT_ACTIVE ))) { - ilog("%s BMC REe-Connect End ; ipmi", node_ptr->hostname.c_str()); + ilog("%s BMC Re-Connect End ; ipmi", node_ptr->hostname.c_str()); /* TODO: may need retries */ - plog ("%s bmc does not support Redfish ; " \ - "defaulting to ipmi", + plog ("%s bmc does not support Redfish", node_ptr->hostname.c_str()); } else { - wlog ("%s %s recv failed (rc:%d:%d:%s); defaulting to ipmi", + wlog ("%s %s recv failed (rc:%d:%d:%s)", node_ptr->hostname.c_str(), bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), rc, node_ptr->bmc_thread_info.status, - node_ptr->bmc_thread_info.status_string.c_str()); + node_ptr->bmc_thread_info.status_string.c_str()); } - node_ptr->bmc_protocol = BMC_PROTOCOL__IPMITOOL ; - mtcInfo_set ( node_ptr, MTCE_INFO_KEY__BMC_PROTOCOL, BMC_PROTOCOL__IPMI_STR ); - mtcInvApi_update_mtcInfo ( node_ptr ); - node_ptr->bmc_protocol_learning = false ; - node_ptr->bmc_thread_ctrl.done = true ; + return ( bmc_default_to_ipmi ( node_ptr ) ); } else { @@ -6691,9 +6683,10 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr ) node_ptr->hostname.c_str(), bmcUtil_getCmd_str( node_ptr->bmc_thread_info.command).c_str()); - node_ptr->bmc_protocol = BMC_PROTOCOL__IPMITOOL ; - mtcInfo_set ( node_ptr, MTCE_INFO_KEY__BMC_PROTOCOL, BMC_PROTOCOL__IPMI_STR ); - mtcInvApi_update_mtcInfo ( node_ptr ); + if ( node_ptr->bmc_protocol_learning ) + { + return ( bmc_default_to_ipmi ( node_ptr ) ); + } } else { @@ -6704,7 +6697,7 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr ) mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_FIRST_WAIT ); } } - else if ( node_ptr->bmc_info_query_active == true ) + else { int rc ; if ( ( rc = bmc_command_recv ( node_ptr ) ) == RETRY ) @@ -6713,10 +6706,16 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr ) } else if ( rc != PASS ) { - /* this error is reported by the receive driver */ - node_ptr->bmc_info_query_active = false ; - node_ptr->bmc_thread_ctrl.done = true ; - mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_BMC_REQUEST_DELAY ); + if ( node_ptr->bmc_protocol_learning ) + bmc_default_to_ipmi ( node_ptr ); + else + { + /* If not in learning mode then force the retry + * from start in MTC_BMC_REQUEST_DELAY seconds */ + bmc_default_query_controls ( node_ptr ); + node_ptr->bmc_thread_ctrl.done = true ; + mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_BMC_REQUEST_DELAY ); + } } else { @@ -6728,67 +6727,229 @@ int nodeLinkClass::bmc_handler ( struct nodeLinkClass::node * node_ptr ) node_ptr->bmc_thread_info.data, node_ptr->bmc_info ) != PASS ) { - elog ("%s bmc %s failed ; defaulting to ipmi", + elog ("%s bmc redfish %s or get bmc info failed", node_ptr->hostname.c_str(), bmcUtil_getCmd_str( node_ptr->bmc_thread_info.command).c_str()); - - node_ptr->bmc_info_query_active = false ; - node_ptr->bmc_info_query_done = false ; - node_ptr->bmc_protocol = BMC_PROTOCOL__IPMITOOL ; - mtcInfo_set ( node_ptr, MTCE_INFO_KEY__BMC_PROTOCOL, BMC_PROTOCOL__IPMI_STR ); + if ( node_ptr->bmc_protocol_learning ) + bmc_default_to_ipmi ( node_ptr ); + else + { + /* If not in learning mode then force the retry + * from start in MTC_BMC_REQUEST_DELAY seconds */ + bmc_default_query_controls ( node_ptr ); + mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_BMC_REQUEST_DELAY ); + } } else { - mtcTimer_reset ( node_ptr->bm_timer ); - mtcTimer_reset ( node_ptr->bmc_audit_timer ); - - int bmc_audit_period = daemon_get_cfg_ptr()->bmc_audit_period ; - if ( bmc_audit_period ) - { - /* the time for the first audit is twice the configured period */ - mtcTimer_start ( node_ptr->bmc_audit_timer, mtcTimer_handler, bmc_audit_period*2 ); - plog ("%s bmc audit timer started (%d secs)", node_ptr->hostname.c_str(), bmc_audit_period*2); - } - else - { - ilog("%s bmc audit disabled", node_ptr->hostname.c_str()); - } - - /* success path */ - node_ptr->bmc_accessible = true ; node_ptr->bmc_info_query_done = true ; node_ptr->bmc_info_query_active = false ; - node_ptr->bmc_protocol_learning = false ; - - mtcInfo_set ( node_ptr, MTCE_INFO_KEY__BMC_PROTOCOL, BMC_PROTOCOL__REDFISH_STR ); - - mtcTimer_reset ( node_ptr->bmc_access_timer ); - - /* save the host's power state */ - node_ptr->power_on = node_ptr->bmc_info.power_on ; - - plog ("%s bmc is accessible using redfish", - node_ptr->hostname.c_str()); - - node_ptr->bmc_thread_ctrl.done = true ; - node_ptr->bmc_thread_info.command = 0 ; } - /* store mtcInfo, which specifies the selected BMC protocol, - * into the sysinv database */ - mtcInvApi_update_mtcInfo ( node_ptr ); - - /* push the BMC access info out to the mtcClient when - * a controller's BMC connection is established/verified */ - if ( node_ptr->nodetype & CONTROLLER_TYPE ) - this->want_mtcInfo_push = true ; - - send_hwmon_command ( node_ptr->hostname, MTC_CMD_ADD_HOST ); - send_hwmon_command ( node_ptr->hostname, MTC_CMD_START_HOST ); } } } + /* Handle Redfish BMC reset/power command query using the redfishtool + * raw GET command. + * This is the last operation before declaring the BMC accessible */ + else if (( node_ptr->bmc_protocol == BMC_PROTOCOL__REDFISHTOOL ) && + ( node_ptr->bmc_accessible == false ) && + ( node_ptr->bm_ping_info.ok == true ) && + ( node_ptr->bmc_info_query_done == true ) && + ( node_ptr->bmc_actions_query_done == false ) && + ( mtcTimer_expired (node_ptr->bm_timer ) == true )) + { + if ( node_ptr->bmc_info.power_ctrl.raw_target_path.empty() ) + { + node_ptr->bmc_actions_query_done = true ; + } + else if ( node_ptr->bmc_actions_query_active == false ) + { + blog ("%s bmc action info target: %s", + node_ptr->hostname.c_str(), + node_ptr->bmc_info.power_ctrl.raw_target_path.c_str()); + + if ( bmc_command_send ( node_ptr, BMC_THREAD_CMD__RAW_GET ) != PASS ) + { + elog ("%s bmc redfish '%s' send failed\n", + node_ptr->hostname.c_str(), + bmcUtil_getCmd_str( + node_ptr->bmc_thread_info.command).c_str()); + if ( node_ptr->bmc_protocol_learning ) + bmc_default_to_ipmi ( node_ptr ); + else + { + /* If not in learning mode then force the retry + * from start in MTC_BMC_REQUEST_DELAY seconds */ + bmc_default_query_controls ( node_ptr ); + mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_BMC_REQUEST_DELAY ); + } + } + else + { + node_ptr->bmc_actions_query_active = true ; + blog ("%s bmc redfish '%s' in progress", + node_ptr->hostname.c_str(), + bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str()); + mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_FIRST_WAIT ); + } + } + else + { + int rc ; + bool default_to_ipmi = false ; + if ( ( rc = bmc_command_recv ( node_ptr ) ) == RETRY ) + { + mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_RETRY_WAIT ); + } + else if ( rc != PASS ) + { + if ( node_ptr->bmc_protocol_learning ) + bmc_default_to_ipmi ( node_ptr ); + else + { + /* If not in learning mode then force the retry + * from start in MTC_BMC_REQUEST_DELAY seconds */ + bmc_default_query_controls ( node_ptr ); + node_ptr->bmc_thread_ctrl.done = true ; + mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_BMC_REQUEST_DELAY ); + } + } + else + { + node_ptr->bmc_thread_ctrl.done = true ; + + blog ("%s bmc thread info cmd: %s data:\n%s", + node_ptr->hostname.c_str(), + bmcUtil_getCmd_str( + node_ptr->bmc_thread_info.command).c_str(), + node_ptr->bmc_thread_info.data.c_str() ); + + /* Look for Parameters as list */ + std::list param_list ; + if ( jsonUtil_get_list ((char*)node_ptr->bmc_thread_info.data.data(), + REDFISH_LABEL__PARAMETERS, + param_list ) == PASS ) + { + /* Walk through the host action list looking for and updating + * this host's bmc_info supported actions lists */ + int index = 0 ; + bool actions_found = false ; + std::list::iterator param_list_ptr ; + for ( param_list_ptr = param_list.begin(); + param_list_ptr != param_list.end() ; + param_list_ptr++, ++index ) + { + std::list action_list ; + string param_list_str = *param_list_ptr ; + blog ("%s %s element %d:%s", + node_ptr->hostname.c_str(), + REDFISH_LABEL__PARAMETERS, + index, param_list_str.c_str()); + + if ( jsonUtil_get_list ((char*)param_list_str.data(), + REDFISH_LABEL__ALLOWABLE_VALUES, + action_list ) == PASS ) + { + actions_found = true ; + redfishUtil_load_actions ( node_ptr->hostname, + node_ptr->bmc_info, + action_list ); + break ; + } + } + if ( actions_found == false ) + { + elog ("%s failed to find '%s' in:\n%s", + node_ptr->hostname.c_str(), + REDFISH_LABEL__ALLOWABLE_VALUES, + node_ptr->bmc_thread_info.data.c_str()); + default_to_ipmi = true ; + } + } + else + { + elog ("%s failed to get Action '%s' list from %s", + node_ptr->hostname.c_str(), + REDFISH_LABEL__PARAMETERS, + node_ptr->bmc_thread_info.data.c_str()); + default_to_ipmi = true ; + } + + /* force failover to use IPMI */ + if ( default_to_ipmi == true ) + { + if ( node_ptr->bmc_protocol_learning ) + bmc_default_to_ipmi ( node_ptr ); + else + { + bmc_default_query_controls ( node_ptr ); + node_ptr->bmc_thread_ctrl.done = true ; + mtcTimer_start ( node_ptr->bm_timer, mtcTimer_handler, MTC_BMC_REQUEST_DELAY ); + } + } + else + { + node_ptr->bmc_actions_query_done = true ; + node_ptr->bmc_actions_query_active = false ; + } + } + } + + /* finish up when the actions query is done */ + if ( node_ptr->bmc_actions_query_done == true ) + { + mtcTimer_reset ( node_ptr->bm_timer ); + mtcTimer_reset ( node_ptr->bmc_audit_timer ); + + int bmc_audit_period = daemon_get_cfg_ptr()->bmc_audit_period ; + if ( bmc_audit_period ) + { + /* the time for the first audit is twice the configured period */ + mtcTimer_start ( node_ptr->bmc_audit_timer, mtcTimer_handler, bmc_audit_period*2 ); + plog ("%s bmc audit timer started (%d secs)", node_ptr->hostname.c_str(), bmc_audit_period*2); + } + else + { + ilog("%s bmc audit disabled", node_ptr->hostname.c_str()); + } + + /* success path */ + node_ptr->bmc_accessible = true ; + node_ptr->bmc_info_query_done = true ; + node_ptr->bmc_info_query_active = false ; + node_ptr->bmc_actions_query_done = true ; + node_ptr->bmc_actions_query_active = false ; + node_ptr->bmc_protocol_learning = false ; + + mtcInfo_set ( node_ptr, MTCE_INFO_KEY__BMC_PROTOCOL, BMC_PROTOCOL__REDFISH_STR ); + + mtcTimer_reset ( node_ptr->bmc_access_timer ); + + /* save the host's power state */ + node_ptr->power_on = node_ptr->bmc_info.power_on ; + + plog ("%s bmc is accessible using redfish", + node_ptr->hostname.c_str()); + + node_ptr->bmc_thread_ctrl.done = true ; + node_ptr->bmc_thread_info.command = 0 ; + + /* store mtcInfo, which specifies the selected BMC protocol, + * into the sysinv database */ + mtcInvApi_update_mtcInfo ( node_ptr ); + + /* push the BMC access info out to the mtcClient when + * a controller's BMC connection is established/verified */ + if ( node_ptr->nodetype & CONTROLLER_TYPE ) + this->want_mtcInfo_push = true ; + + send_hwmon_command ( node_ptr->hostname, MTC_CMD_ADD_HOST ); + send_hwmon_command ( node_ptr->hostname, MTC_CMD_START_HOST ); + } + } + /***************************************************************** * Handle IPMI BMC Info Query * diff --git a/mtce/src/maintenance/mtcThreads.cpp b/mtce/src/maintenance/mtcThreads.cpp index 7fef9df2..97db3af5 100644 --- a/mtce/src/maintenance/mtcThreads.cpp +++ b/mtce/src/maintenance/mtcThreads.cpp @@ -109,6 +109,12 @@ void * mtcThread_bmc ( void * arg ) suffix = BMC_QUERY_FILE_SUFFIX ; break ; } + case BMC_THREAD_CMD__RAW_GET: + { + command = extra_ptr->bm_cmd ; + suffix = BMC_RAW_GET_FILE_SUFFIX ; + break ; + } case BMC_THREAD_CMD__BMC_INFO: case BMC_THREAD_CMD__POWER_STATUS: {