/* * Copyright (c) 2013-2016 Wind River Systems, Inc. * * SPDX-License-Identifier: Apache-2.0 * */ /** * @file * Wind River CGTS Platform Node Maintenance "Compute Messaging" * Implementation */ /** * @detail * Detailed description ... * * */ #include #include #include /* for ... unix domain sockets */ #include #include #include #include #include /* for hostent */ #include #include #include #include #include #include /* for close and usleep */ #include #include #include using namespace std; #define __AREA__ "msg" #include "nodeClass.h" #include "mtcNodeMsg.h" #include "jsonUtil.h" /* for ... jsonApi_get_key_value */ #include "daemon_option.h" #include "daemon_common.h" #include "mtcAlarm.h" /* for ... mtcAlarm... */ #include "nodeUtil.h" /* for ... get_event_str ... */ int service_events ( nodeLinkClass * obj_ptr, mtc_socket_type * sock_ptr ); /* Throttle logging of messages from unknown IP addresses */ std::list unknown_ip_list ; /* Send specified command to the guestAgent daemon */ int send_guest_command ( string hostname, int command ) { int rc = PASS ; nodeLinkClass * obj_ptr = get_mtcInv_ptr(); mlog ("%s NodeType %s (0x%x) Check: %c %c \n", hostname.c_str(), obj_ptr->functions.c_str(), obj_ptr->get_nodetype(hostname), obj_ptr->is_compute (hostname) ? 'Y' : 'n', obj_ptr->is_compute_subfunction(hostname) ? 'Y' : 'n'); if ( obj_ptr->is_compute (hostname) || obj_ptr->is_compute_subfunction(hostname)) { mtc_message_type msg ; /* the message to send */ int bytes_to_send; int bytes ; mtc_socket_type * sock_ptr = get_sockPtr (); ilog ("%s sending '%s' to guest service\n", hostname.c_str(), get_event_str(command).c_str()); string hostinfo = "{\"hostname\":\"" ; hostinfo.append (hostname); hostinfo.append ("\""); if ( (command == MTC_CMD_ADD_HOST) || (command == MTC_CMD_MOD_HOST) ) { hostinfo.append (",\"uuid\":\""); hostinfo.append ( obj_ptr->get_uuid (hostname)); hostinfo.append ( "\""); hostinfo.append (",\"ip\":\""); hostinfo.append ( obj_ptr->get_hostaddr (hostname)); hostinfo.append ( "\""); hostinfo.append (",\"personality\":\"compute\""); } else if ( command == MTC_CMD_DEL_HOST ) { hostinfo.append (",\"uuid\":\""); hostinfo.append ( obj_ptr->get_uuid (hostname)); hostinfo.append ( "\""); } hostinfo.append ( "}"); /* Add the header, command and the host info to the message */ memset (&msg, 0, sizeof(mtc_message_type)); memcpy(&msg.hdr[0], get_cmd_req_msg_header(), MSG_HEADER_SIZE ); memcpy(&msg.buf[0], hostinfo.data(), hostinfo.length()); msg.cmd = command ; /* Send to guestAgent daemon port */ bytes_to_send = ((sizeof(mtc_message_type))-(BUF_SIZE)+(hostinfo.length())); bytes = sock_ptr->mtc_to_hbs_sock->write((char*) &msg, bytes_to_send, NULL, daemon_get_cfg_ptr()->mtc_to_guest_cmd_port); if ( bytes <= 0 ) { wlog ("Cannot send to heartbeat service\n"); rc = FAIL_TO_TRANSMIT ; } } return (rc); } /* Log throttle counters for this interface */ int rx_error_count = 0 ; int mtc_service_inbox ( nodeLinkClass * obj_ptr, mtc_socket_type * sock_ptr, int iface) { mtc_message_type msg ; int bytes = 0 ; int rc = PASS ; if ( iface == INFRA_INTERFACE ) { if ( ( obj_ptr ) && ( obj_ptr->infra_network_provisioned == true ) && ( sock_ptr->mtc_agent_infra_rx_socket )) { bytes = sock_ptr->mtc_agent_infra_rx_socket->read((char*)&msg, sizeof(msg)); } else { return ( FAIL_NO_INFRA_PROV ); } } else { bytes = sock_ptr->mtc_agent_rx_socket->read((char*)&msg, sizeof(msg)); } msg.buf[BUF_SIZE-1] = '\0'; if ( bytes <= 0 ) { return (RETRY); } else if ( bytes < 7 ) { wlog_throttled ( rx_error_count, 100, "Message receive error, underrun (only rxed %d bytes)\n", bytes ); return (FAIL); } else { rx_error_count = 0 ; } zero_unused_msg_buf (msg, bytes); string hostaddr = "" ; string hostname = "" ; if ( iface == INFRA_INTERFACE ) { hostaddr = sock_ptr->mtc_agent_infra_rx_socket->get_src_str(); hostname = obj_ptr->get_hostname ( hostaddr ) ; } else { hostaddr = sock_ptr->mtc_agent_rx_socket->get_src_str(); hostname = obj_ptr->get_hostname ( hostaddr ) ; } if ( hostname.empty() ) { std::list::iterator iter ; iter = std::find (unknown_ip_list.begin(), unknown_ip_list.end(), hostaddr ); if ( iter == unknown_ip_list.end() ) { mlog3 ( "Received message from unknown IP <%s>\n", hostaddr.c_str()); unknown_ip_list.push_front(hostaddr); } return (FAIL_NOT_FOUND); } else if ( ! hostaddr.empty() ) { unknown_ip_list.remove (hostaddr); } print_mtc_message ( hostname, MTC_CMD_RX, msg, get_iface_name_str(iface), false ); if ( msg.hdr[0] == '{' ) { int rc1 ; string service ; mlog1 ("%s\n", &msg.hdr[0] ); rc1 = jsonUtil_get_key_val(&msg.hdr[0],"service", service ); if ( rc1 == PASS ) { if ( service == "collectd_notifier" ) { int rc1,rc2,rc3 ; string hostname,resource,state ; rc1 = jsonUtil_get_key_val(&msg.hdr[0],"hostname", hostname ); rc2 = jsonUtil_get_key_val(&msg.hdr[0],"resource", resource ); rc3 = jsonUtil_get_key_val(&msg.hdr[0],"degrade", state ); if ( rc1|rc2|rc3 ) { elog ("failed to parse '%s' message\n", service.c_str()); wlog ("... %s\n", &msg.hdr[0] ); } else { obj_ptr->collectd_notify_handler ( hostname, resource, state ); } } /* future service requests */ else { wlog ("Unexpected service request: '%s'\n", service.c_str()); } } else { wlog("Unexpected json message: %s\n", &msg.hdr[0] ); } } /* Check for response messages */ else if ( strstr ( &msg.hdr[0], get_cmd_rsp_msg_header() ) ) { obj_ptr->set_cmd_resp ( hostname , msg ) ; } /* * Check for compute messages */ else if ( strstr ( &msg.hdr[0], get_compute_msg_header() ) ) { if ( msg.cmd == MTC_MSG_MTCALIVE ) { string functions = "" ; rc = jsonUtil_get_key_val ( &msg.buf[0], "personality", functions ); if ( rc ) { wlog ("%s failed to get personality from mtcAlive message\n", hostname.c_str()); return (FAIL_KEY_VALUE_PARSE); } rc = obj_ptr->update_host_functions ( hostname, functions ); dlog3 ("%s functions: %s\n", hostname.c_str(), functions.c_str()); if ( rc ) { wlog ("%s failed to load functions from mtcAlive message\n", hostname.c_str()); return (FAIL_NODETYPE); } obj_ptr->set_uptime ( hostname , msg.parm[MTC_PARM_UPTIME_IDX], false ); obj_ptr->set_health ( hostname , msg.parm[MTC_PARM_HEALTH_IDX] ); obj_ptr->set_mtce_flags ( hostname , msg.parm[MTC_PARM_FLAGS_IDX] ); obj_ptr->set_mtcAlive ( hostname, iface ); mlog1("%s Uptime:%d Health:%d Flags:0x%x mtcAlive:%s\n", hostname.c_str(), msg.parm[MTC_PARM_UPTIME_IDX], msg.parm[MTC_PARM_HEALTH_IDX], msg.parm[MTC_PARM_FLAGS_IDX], obj_ptr->get_mtcAlive_gate ( hostname ) ? "gated" : "open"); string infra_ip = ""; /* Get the infra ip address if it is provisioned */ rc = jsonUtil_get_key_val ( &msg.buf[0], "infra_ip", infra_ip ); if ( rc == PASS ) { obj_ptr->set_infra_hostaddr ( hostname, infra_ip ); } else { mlog ("%s null or missing 'infra_ip' value (rc:%d)\n", hostname.c_str(), rc); } } else if ( msg.cmd == MTC_MSG_MAIN_GOENABLED ) { if ( !obj_ptr->my_hostname.compare(hostname) ) { ilog ("%s received GOENABLED from self\n", hostname.c_str()); } rc = send_mtc_cmd ( hostname , msg.cmd, MGMNT_INTERFACE ); if ( rc != PASS ) { elog ("%s GOENABLED send reply failed (rc:%d)\n", hostname.c_str(), rc); wlog ("%s ... need successful GOENABLED reply, dropping ...\n", hostname.c_str() ); } else { mlog ("%s got GOENABLED (out-of-service tests passed) message\n", hostname.c_str()); obj_ptr->set_goEnabled ( hostname ); } } else if ( msg.cmd == MTC_MSG_MAIN_GOENABLED_FAILED ) { if ( obj_ptr->get_adminState ( hostname ) == MTC_ADMIN_STATE__UNLOCKED ) { wlog ("%s failed out-of-service test: %s\n", hostname.c_str(), &msg.buf[0] ); obj_ptr->set_goEnabled_failed ( hostname ); } /* We don't send a reply on a fail */ } else if ( msg.cmd == MTC_MSG_SUBF_GOENABLED ) { mlog ("%s-compute GOENABLED message\n", hostname.c_str()); if ( !obj_ptr->my_hostname.compare(hostname) ) { ilog ("%s-compute received GOENABLED from self\n", hostname.c_str()); } rc = send_mtc_cmd ( hostname , msg.cmd, MGMNT_INTERFACE ); if ( rc != PASS ) { elog ("%s-compute GOENABLED send reply failed (rc:%d)\n", hostname.c_str(), rc); wlog ("%s-compute ... need successful GOENABLED reply, dropping ...\n", hostname.c_str() ); } else { mlog ("%s-compute got GOENABLED (out-of-service tests passed) message\n", hostname.c_str()); obj_ptr->set_goEnabled_subf ( hostname ); } } else if ( msg.cmd == MTC_MSG_SUBF_GOENABLED_FAILED ) { if ( obj_ptr->get_adminState ( hostname ) == MTC_ADMIN_STATE__UNLOCKED ) { wlog ("%s-compute failed GOENABLE test: %s\n", hostname.c_str(), &msg.buf[0] ); obj_ptr->set_goEnabled_failed_subf ( hostname ); } /* We don't send a reply on a fail */ } else { wlog ("Unexpected compute message (0x%x) from '%s'\n", msg.cmd, hostname.c_str()); } } /* * Check for Event Messages */ else if ( strstr ( &msg.hdr[0], get_mtce_event_header() ) ) { rc = PASS ; if ( hostname.empty() ) { mlog2 ( "Received mtce event from unknown host\n"); rc = FAIL_UNKNOWN_HOSTNAME ; } else if ( !hostname.compare("localhost") ) { mlog2 ("localhost event (%x) ignored", msg.cmd); } else { string event = "" ; /* TODO: fix this hostname setting */ if (( msg.cmd == MTC_DEGRADE_CLEAR ) || ( msg.cmd == MTC_DEGRADE_RAISE ) || ( msg.cmd == MTC_EVENT_HWMON_CLEAR ) || ( msg.cmd == MTC_EVENT_HWMON_MINOR ) || ( msg.cmd == MTC_EVENT_HWMON_MAJOR ) || ( msg.cmd == MTC_EVENT_HWMON_CRIT ) || ( msg.cmd == MTC_EVENT_HWMON_RESET ) || ( msg.cmd == MTC_EVENT_HWMON_POWERDOWN ) || ( msg.cmd == MTC_EVENT_HWMON_POWERCYCLE) || ( msg.cmd == MTC_EVENT_HWMON_CONFIG )) { hostname = &msg.hdr[MSG_HEADER_SIZE] ; } /* the mtce event (process or resource) that causes this raised event is at the * head of the message buffer. Load it into an 'event' * string to be passed into the individual handlers for * convenience. Safer to pass reference to a string than * the raw buffer pointer. */ if ( strnlen ( &msg.buf[0] , MAX_MTCE_EVENT_NAME_LEN ) ) { event = msg.buf ; } switch ( msg.cmd ) { /* TODO: Port other services to use this common code */ case MTC_EVENT_MONITOR_READY: { std::list::iterator temp ; // bool start_monitoring_flag = false ; if ( !event.compare("pmond") ) { /* Notify mtcAgent that we got a pmond ready event */ obj_ptr->declare_service_ready ( hostname, MTC_SERVICE_PMOND ); return (PASS); } else if ( !event.compare("hbsClient") ) { /* Notify mtcAgent that we got a hbsClient ready event */ obj_ptr->declare_service_ready ( hostname, MTC_SERVICE_HEARTBEAT ); return (PASS); } /* If the active controller got the ready event from a local service * then push the inventory to that service and for each host that is * enabled send the start monitoring command to it if the bm_ip is * provisioned. * Handles the daemon restart case */ for ( temp = obj_ptr->hostname_inventory.begin () ; temp != obj_ptr->hostname_inventory.end () ; temp++ ) { hostname = temp->data(); /* Set the general start monitoring flag based on service state. * This lag may be over ridden my individual services based on * additional information */ if (( obj_ptr->get_adminState ( hostname ) == MTC_ADMIN_STATE__UNLOCKED ) && ( obj_ptr->get_operState ( hostname ) == MTC_OPER_STATE__ENABLED ) && ((obj_ptr->get_availStatus ( hostname ) == MTC_AVAIL_STATUS__AVAILABLE ) || (obj_ptr->get_availStatus ( hostname ) == MTC_AVAIL_STATUS__DEGRADED ))) { ; // start_monitoring_flag = true ; } else { ; // start_monitoring_flag = false ; } if ( !event.compare("hwmond") ) { obj_ptr->declare_service_ready ( hostname, MTC_SERVICE_HWMOND ); } else { wlog ("%s Global Ready Event not supported for '%s' service\n", hostname.c_str(), event.c_str()); return (FAIL_BAD_PARM); } } break ; } /***************************************************************** * Data Port Events * *****************************************************************/ /***************************************************************** * Process Monitor Events * *****************************************************************/ case MTC_EVENT_PMON_CLEAR: { mlog ("%s pmond: '%s' recovered (clear)\n", hostname.c_str(), event.c_str()); obj_ptr->degrade_pmond_clear ( hostname ); break ; } case MTC_EVENT_PMON_CRIT: { mlog ("%s pmond: '%s' failed (critical)\n", hostname.c_str(), event.c_str()); /** * event is the process name that has failed * parm[0] is the nodetype the process serves **/ obj_ptr->critical_process_failed ( hostname, event, msg.parm[0] ); break ; } case MTC_EVENT_PMON_MAJOR: { mlog ("%s pmond: '%s' failed (major)\n", hostname.c_str(), event.c_str()); obj_ptr->degrade_process_raise ( hostname, event ); break ; } case MTC_EVENT_PMON_MINOR: { mlog ("%s pmond: '%s' failed (minor)\n", hostname.c_str(), event.c_str()); obj_ptr->alarm_process_failure ( hostname, event ); break ; } case MTC_EVENT_PMON_LOG: { mlog ("%s pmond: '%s' failed (log)\n", hostname.c_str(), event.c_str()); obj_ptr->log_process_failure ( hostname, event ); break ; } /***************************************************************** * Resource Monitor Events * *****************************************************************/ /* TODO: Remove - Suspecting OBS Command */ case MTC_EVENT_RMON_READY: { mlog ("%s RMON Ready\n", hostname.c_str()); obj_ptr->declare_service_ready ( hostname, MTC_SERVICE_RMOND ); break ; } case MTC_EVENT_RMON_CLEAR: { mlog ("%s rmond: '%s' recovered (clear)\n", hostname.c_str(), event.c_str()); obj_ptr->degrade_resource_clear ( hostname , event ); break ; } case MTC_EVENT_RMON_CRIT: { mlog ("%s rmond: '%s' failed (critical)\n", hostname.c_str(), event.c_str()); obj_ptr->critical_resource_failed ( hostname, event ); break ; } case MTC_EVENT_RMON_MAJOR: { mlog ("%s rmond: '%s' failed (major)\n", hostname.c_str(), event.c_str()); obj_ptr->degrade_resource_raise ( hostname, event ); break ; } case MTC_EVENT_RMON_MINOR: { mlog ("%s rmond: '%s' failed (minor)\n", hostname.c_str(), event.c_str()); /* Clear the degrade condition if one is present */ obj_ptr->degrade_resource_clear ( hostname , event ); obj_ptr->log_resource_failure ( hostname, event ); break ; } case MTC_EVENT_HWMON_CLEAR: case MTC_DEGRADE_CLEAR: { mlog ("%s hwmon requests to clear its degrade flag\n", hostname.c_str()); obj_ptr->node_degrade_control ( hostname, MTC_DEGRADE_CLEAR , "hwmon" ); break ; } case MTC_EVENT_HWMON_MINOR: case MTC_EVENT_HWMON_MAJOR: case MTC_EVENT_HWMON_CRIT: case MTC_DEGRADE_RAISE: { mlog ("%s hwmon requested to set its degrade flag\n", hostname.c_str()); obj_ptr->node_degrade_control ( hostname, MTC_DEGRADE_RAISE , "hwmon" ); break ; } case MTC_EVENT_HWMON_RESET: case MTC_EVENT_HWMON_POWERDOWN: case MTC_EVENT_HWMON_POWERCYCLE: { mlog ("%s requires maintenance '%s' action due to failing '%s' sensor \n", hostname.c_str(), get_event_str(msg.cmd).c_str(), event.c_str()); obj_ptr->invoke_hwmon_action ( hostname, msg.cmd, event ); break ; } default: { wlog ("%s Unknown Event (%x)\n", hostname.c_str(), msg.cmd ); rc = FAIL ; break ; } } } } else { wlog ( "Received unsupported or badly formed message\n" ); } /* Only do this if the debug level is appropriate */ if ( daemon_get_cfg_ptr()->debug_msg ) { int count = 0 ; std::list::iterator iter ; for ( iter = unknown_ip_list.begin () ; iter != unknown_ip_list.end () ; iter++ ) { count++ ; mlog3 ("Unknown IP [%d]:%s\n", count, iter->c_str()); } } return (rc); } int send_mtc_cmd ( string & hostname, int cmd , int interface ) { int rc = FAIL ; bool force = false ; mtc_message_type mtc_cmd ; mtc_socket_type * sock_ptr = get_sockPtr (); memset (&mtc_cmd,0,sizeof(mtc_message_type)); /* Add the command version to he message */ mtc_cmd.ver = MTC_CMD_VERSION ; mtc_cmd.rev = MTC_CMD_REVISION; switch ( cmd ) { case MTC_REQ_MTCALIVE: { snprintf ( &mtc_cmd.hdr[0], MSG_HEADER_SIZE, "%s" , get_cmd_req_msg_header() ); mtc_cmd.cmd = cmd ; mtc_cmd.num = 0 ; rc = PASS ; break ; } case MTC_REQ_MAIN_GOENABLED: case MTC_REQ_SUBF_GOENABLED: { snprintf ( &mtc_cmd.hdr[0], MSG_HEADER_SIZE, "%s", get_cmd_req_msg_header() ); mtc_cmd.cmd = cmd ; mtc_cmd.num = 0 ; rc = PASS ; break ; } case MTC_CMD_STOP_CONTROL_SVCS: case MTC_CMD_STOP_COMPUTE_SVCS: case MTC_CMD_STOP_STORAGE_SVCS: case MTC_CMD_START_CONTROL_SVCS: case MTC_CMD_START_COMPUTE_SVCS: case MTC_CMD_START_STORAGE_SVCS: { snprintf ( &mtc_cmd.hdr[0], MSG_HEADER_SIZE, "%s", get_cmd_req_msg_header() ); mtc_cmd.cmd = cmd ; rc = PASS ; break ; } case MTC_CMD_RESET: case MTC_CMD_REBOOT: case MTC_CMD_WIPEDISK: case MTC_CMD_LAZY_REBOOT: { ilog ("%s sending '%s' request (%s network)\n", hostname.c_str(), get_mtcNodeCommand_str(cmd), get_iface_name_str(interface)); snprintf ( &mtc_cmd.hdr[0], MSG_HEADER_SIZE, "%s", get_cmd_req_msg_header() ); mtc_cmd.cmd = cmd ; mtc_cmd.num = 0 ; if ( cmd == MTC_CMD_LAZY_REBOOT ) { mtc_cmd.num = 1 ; mtc_cmd.parm[0] = MTC_SECS_30 ; } rc = PASS ; break ; } /* Tell the mtcClient on that host that its subFunction has failed */ case MTC_MSG_SUBF_GOENABLED_FAILED: { force = true ; ilog ("%s sending '%s' request (%s network)\n", hostname.c_str(), get_mtcNodeCommand_str(cmd), get_iface_name_str(interface)); snprintf ( &mtc_cmd.hdr[0], MSG_HEADER_SIZE, "%s", get_cmd_req_msg_header() ); mtc_cmd.cmd = cmd ; mtc_cmd.num = 0 ; rc = PASS ; break ; } case MTC_MSG_MTCALIVE: { slog ("request to send mtcAlive message from mtcAgent ; invalid\n"); return (FAIL_OPERATION); } case MTC_MSG_MAIN_GOENABLED: case MTC_MSG_SUBF_GOENABLED: { snprintf ( &mtc_cmd.hdr[0], MSG_HEADER_SIZE, "%s", get_msg_rep_msg_header() ); mtc_cmd.cmd = cmd ; mtc_cmd.num = 0 ; rc = PASS ; break ; } case MTC_MSG_LOCKED: { mlog ("%s sending 'Locked' notification (%s network)\n", hostname.c_str(), get_iface_name_str(interface)); snprintf ( &mtc_cmd.hdr[0], MSG_HEADER_SIZE, "%s", get_cmd_req_msg_header() ); mtc_cmd.cmd = cmd ; mtc_cmd.num = 0 ; rc = PASS ; break ; } default: { elog ("Unsupported maintenance command (0x%x)\n", cmd ); rc = FAIL_BAD_CASE ; } } if ( rc == PASS ) { int bytes = 0; /* Temporarily get IP from node inventory till dns is available */ nodeLinkClass * obj_ptr = get_mtcInv_ptr (); /* add the mac address of the target card to the header * Note: the minus 1 is to overwqrite the null */ snprintf ( &mtc_cmd.hdr[MSG_HEADER_SIZE-1], MSG_HEADER_SIZE, "%s", obj_ptr->get_hostIfaceMac(hostname, MGMNT_IFACE).data()); /* Lets add the controller's floating ip in the buffer so hat he host knowns where to reply */ snprintf ( &mtc_cmd.buf[0], obj_ptr->my_float_ip.length()+1, "%s", obj_ptr->my_float_ip.data()); /* only send the minimum amount of data */ bytes = (sizeof(mtc_message_type)-(BUF_SIZE-(obj_ptr->my_float_ip.length()+1))) ; print_mtc_message ( hostname, MTC_CMD_TX, mtc_cmd, get_iface_name_str(interface), force ) ; if (interface == MGMNT_INTERFACE) { string hostaddr = obj_ptr->get_hostaddr(hostname); #ifdef WANT_FIT_TESTING if ( daemon_want_fit ( FIT_CODE__INVALIDATE_MGMNT_IP, hostname ) ) hostaddr = "none" ; #endif if ( hostUtil_is_valid_ip_addr ( hostaddr ) != true ) { wlog("%s has no management IP assigned\n", hostname.c_str()); return (FAIL_HOSTADDR_LOOKUP); } /* rc = message size */ rc = sock_ptr->mtc_agent_tx_socket->write((char *)&mtc_cmd, bytes, hostaddr.c_str(), sock_ptr->mtc_cmd_port); } else if ((interface == INFRA_INTERFACE) && ( obj_ptr->infra_network_provisioned == true ) && ( sock_ptr->mtc_agent_infra_tx_socket != NULL )) { /* SETUP TX -> COMPUTE SOCKET INFRA INTERFACE */ string infra_hostaddr = obj_ptr->get_infra_hostaddr(hostname); #ifdef WANT_FIT_TESTING if ( daemon_want_fit ( FIT_CODE__INVALIDATE_INFRA_IP, hostname ) ) infra_hostaddr = "none" ; #endif if ( hostUtil_is_valid_ip_addr( infra_hostaddr ) != true ) { return (FAIL_NO_INFRA_PROV); } rc = sock_ptr->mtc_agent_infra_tx_socket->write((char *)&mtc_cmd, bytes, infra_hostaddr.c_str(), sock_ptr->mtc_cmd_port); } if ( 0 > rc ) { elog("%s Failed to send command (rc:%i)\n", hostname.c_str(), rc); rc = FAIL_SOCKET_SENDTO ; } else { rc = PASS ; } } return ( rc ); } int send_hbs_command ( string hostname, int cmd, string controller ) { int bytes = 0 ; int bytes_to_send = 0 ; int rc = PASS ; nodeLinkClass * obj_ptr = get_mtcInv_ptr () ; mtc_message_type event ; mtc_socket_type * sock_ptr = get_sockPtr (); memset (&event, 0 , sizeof(mtc_message_type)); snprintf ( &event.hdr[0] , MSG_HEADER_SIZE, "%s", get_hbs_cmd_req_header() ); snprintf ( &event.hdr[MSG_HEADER_SIZE] , MAX_CHARS_HOSTNAME , "%s", hostname.data()); /* There is no buffer data in any of these messages */ bytes_to_send = ((sizeof(mtc_message_type))-(BUF_SIZE)) ; event.cmd = cmd ; event.num = 1 ; event.parm[0] = obj_ptr->get_nodetype(hostname); /* send to hbsAgent daemon port */ std::list controllers ; controllers.clear(); if ( controller == CONTROLLER ) { controllers.push_back(CONTROLLER_0); controllers.push_back(CONTROLLER_1); } else { controllers.push_back(controller); } string ip = "" ; std::list::iterator unit ; for ( unit = controllers.begin () ; unit != controllers.end () ; unit++ ) { switch ( cmd ) { case MTC_CMD_ACTIVE_CTRL: mlog3 ("%s sending 'activity state' to %s heartbeat service\n", hostname.c_str(), unit->c_str()); break ; case MTC_CMD_STOP_HOST: ilog ("%s sending 'stop' to %s heartbeat service\n", hostname.c_str(), unit->c_str()); break ; case MTC_CMD_START_HOST: obj_ptr->manage_heartbeat_clear ( hostname , MAX_IFACES ); ilog ("%s sending 'start' to %s heartbeat service\n", hostname.c_str(), unit->c_str()); break ; case MTC_CMD_DEL_HOST: ilog ("%s sending 'delete' to %s heartbeat service\n", hostname.c_str(), unit->c_str()); break ; case MTC_CMD_ADD_HOST: obj_ptr->manage_heartbeat_clear ( hostname, MAX_IFACES ); ilog ("%s sending 'add' to %s heartbeat service\n", hostname.c_str(), unit->c_str()); break ; case MTC_RESTART_HBS: ilog ("%s sending 'restart' to %s heartbeat service\n", hostname.c_str(), unit->c_str()); break ; case MTC_BACKOFF_HBS: ilog ("%s requesting %s heartbeat period backoff\n", hostname.c_str(), unit->c_str()); break ; case MTC_RECOVER_HBS: ilog ("%s requesting %s heartbeat period recovery\n", hostname.c_str(), unit->c_str()); break ; default: { slog ("%s Unsupported command operation 0x%x\n", hostname.c_str(), cmd ); rc = FAIL_BAD_PARM ; continue ; } } ip = get_mtcInv_ptr()->get_hostaddr(*unit) ; bytes = sock_ptr->mtc_to_hbs_sock->write((char*) &event, bytes_to_send, ip.data()); if ( bytes <= 0 ) { wlog ("%s failed to send command (0x%x) to heartbeat service at %s\n", unit->c_str(), cmd, ip.c_str() ); rc = FAIL_TO_TRANSMIT ; } } return rc ; } /* Handle client 'events' */ int service_events ( nodeLinkClass * obj_ptr, mtc_socket_type * sock_ptr ) { mtc_message_type msg ; int bytes = 0 ; /* Receive event messages */ memset (&msg, 0, sizeof(mtc_message_type)); bytes = sock_ptr->mtc_event_rx_sock->read((char*)&msg, sizeof(mtc_message_type)); if ( bytes <= 0 ) { return (RETRY) ; } if ( msg.cmd == MTC_EVENT_LOOPBACK ) { const char * event_hdr_ptr = get_loopback_header() ; /* Confirm header */ if ( strncmp ( &msg.hdr[0], event_hdr_ptr, MSG_HEADER_SIZE ) ) { elog ("Invalid Event header\n"); } else { ilog ("Service ping\n"); /* Should send back a response */ } } else if (( msg.cmd == MTC_EVENT_HEARTBEAT_MINOR_SET ) || ( msg.cmd == MTC_EVENT_HEARTBEAT_MINOR_CLR )) { const char * event_hdr_ptr = get_heartbeat_event_header() ; /* Confirm header */ if ( strncmp ( &msg.hdr[0], event_hdr_ptr, MSG_HEADER_SIZE ) ) { elog ("Invalid Heartbeat Event header\n"); } else { string hostname = &msg.buf[0] ; print_mtc_message ( hostname, MTC_CMD_RX, msg, get_iface_name_str(MGMNT_INTERFACE), false ); /* The interface that the heartbeat loss occurred over is * specified in parm[0 for this command * 0 = MGMNT_IFACE * 1 = INFRA_IFACE * else default to 0 (MGMNT_IFACE) to be backwards compatible * * */ iface_enum iface = MGMNT_IFACE; if ( msg.num > 0 ) { if ( msg.parm[0] == INFRA_IFACE ) { iface = INFRA_IFACE ; } } if ( msg.cmd == MTC_EVENT_HEARTBEAT_MINOR_SET ) { /* Assert the minor condition with the 'false' (i.e. not clear)*/ obj_ptr->manage_heartbeat_minor ( hostname, iface, false ); } else { /* Clear the minor condition with the 'clear=true' */ obj_ptr->manage_heartbeat_minor ( hostname, iface, true ); } } } else if (( msg.cmd == MTC_EVENT_HEARTBEAT_DEGRADE_SET ) || ( msg.cmd == MTC_EVENT_HEARTBEAT_DEGRADE_CLR )) { const char * event_hdr_ptr = get_heartbeat_event_header() ; /* Confirm header */ if ( strncmp ( &msg.hdr[0], event_hdr_ptr, MSG_HEADER_SIZE ) ) { elog ("Invalid Heartbeat Event header\n"); } else { string hostname = &msg.buf[0] ; print_mtc_message ( hostname, MTC_CMD_RX, msg, get_iface_name_str(MGMNT_INTERFACE), false ); /* The interface that the heartbeat loss occurred over is * specified in parm[0 for this command * 0 = MGMNT_IFACE * 1 = INFRA_IFACE * else default to 0 (MGMNT_IFACE) to be backwards compatible * * */ iface_enum iface = MGMNT_IFACE; if ( msg.num > 0 ) { if ( msg.parm[0] == INFRA_IFACE ) { iface = INFRA_IFACE ; } } if ( msg.cmd == MTC_EVENT_HEARTBEAT_DEGRADE_SET ) { if (( obj_ptr->hbs_failure_action == HBS_FAILURE_ACTION__FAIL ) || ( obj_ptr->hbs_failure_action == HBS_FAILURE_ACTION__DEGRADE )) { /* Assert the degrade condition with the 'false' (i.e. not clear)*/ obj_ptr->manage_heartbeat_degrade ( hostname, iface, false ); } /* Otherwise the action must be alarm only or none ; both of which * are already handled by the hbsAgent, so do nothing */ else { ilog ("%s heartbeat degrade event dropped ; action is not fail or degrade (%s)\n", hostname.c_str(), get_iface_name_str(iface)); } } else { /* Clear the degrade condition with the 'true' */ obj_ptr->manage_heartbeat_degrade ( hostname, iface, true ); } } } else if ( msg.cmd == MTC_EVENT_HEARTBEAT_LOSS ) { const char * loss_hdr_ptr = get_heartbeat_loss_header() ; /* Confirm header */ if ( strncmp ( &msg.hdr[0], loss_hdr_ptr, MSG_HEADER_SIZE ) ) { elog ("Invalid Heartbeat Loss event header\n"); } else { /* The interface that the heartbeat loss occurred over is * specified in parm[0 for this command * 0 = MGMNT_IFACE * 1 = INFRA_IFACE * else default to 0 (MGMNT_IFACE) to be backwards compatible * * */ iface_enum iface = MGMNT_IFACE; if ( msg.num > 0 ) { if ( msg.parm[0] == INFRA_IFACE ) { iface = INFRA_IFACE ; } } string hostname = &msg.buf[0] ; print_mtc_message ( hostname, MTC_CMD_RX, msg, get_iface_name_str(MGMNT_INTERFACE), false ); /* If heartbeat failure action is fail then call the fail handler */ if ( obj_ptr->hbs_failure_action == HBS_FAILURE_ACTION__FAIL ) obj_ptr->manage_heartbeat_failure ( hostname, iface, false ); /* If heartbeat failure action is degrade then call the degrade handler */ else if ( obj_ptr->hbs_failure_action == HBS_FAILURE_ACTION__DEGRADE ) obj_ptr->manage_heartbeat_degrade ( hostname, iface, false ); /* Otherwise the action must be alarm only or none ; both of which * are already handled by the hbsAgent, so do nothing */ else { ilog ("%s heartbeat loss event dropped ; action is not fail or degrade (%s)\n", hostname.c_str(), get_iface_name_str(iface)); } } } else if ( msg.cmd == MTC_EVENT_PMOND_CLEAR ) { string hostname = &msg.hdr[MSG_HEADER_SIZE] ; string process = "pmond" ; ilog ("%s Degrade Clear Event for process '%s'\n", hostname.c_str(), process.c_str()); print_mtc_message ( hostname, MTC_CMD_RX, msg, get_iface_name_str(MGMNT_INTERFACE), false ); obj_ptr->degrade_pmond_clear ( hostname ); } else if ( msg.cmd == MTC_EVENT_PMOND_RAISE ) { string hostname = &msg.hdr[MSG_HEADER_SIZE] ; string process = "pmond" ; ilog ("%s Degrade Assert Event for process '%s'\n", hostname.c_str(), process.c_str()); print_mtc_message ( hostname, MTC_CMD_RX, msg, get_iface_name_str(MGMNT_INTERFACE), false ); obj_ptr->degrade_process_raise ( hostname , process ); } else if ( msg.cmd == MTC_EVENT_HOST_STALLED ) { string hostname = &msg.hdr[MSG_HEADER_SIZE] ; print_mtc_message ( hostname, MTC_CMD_RX, msg, get_iface_name_str(MGMNT_INTERFACE), false ); elog ("%s Stalled !!!\n", hostname.c_str()); } else if ( msg.cmd == MTC_EVENT_MONITOR_READY ) { string daemon = &msg.hdr[MSG_HEADER_SIZE] ; if ( !daemon.compare("guestAgent") ) { std::list::iterator temp ; int rc = PASS ; /* If the active controller got the ready event from a local service * then push the inventory to that service and for each host that is * enabled send the start monitoring command to it. * Handles the daemon restart case */ for ( temp = obj_ptr->hostname_inventory.begin () ; temp != obj_ptr->hostname_inventory.end () ; temp++ ) { string hostname = temp->data(); rc = send_guest_command ( hostname, MTC_CMD_ADD_HOST ); if ( rc ) { elog ("%s host add to '%s' failed\n", hostname.c_str(), daemon.c_str()); } else { ilog ("%s added to guestAgent\n", hostname.c_str()); } } /* Done sending the host info */ } else { wlog ("Unsupported ready event for daemon: '%s'\n", daemon.c_str()); } } else if ( msg.cmd == MTC_EVENT_HEARTBEAT_READY ) { string controller = CONTROLLER ; std::list::iterator temp ; /* no heartbeating in simplex mode */ if ( obj_ptr->system_type == SYSTEM_TYPE__CPE_MODE__SIMPLEX ) { return (PASS); } /* get the controller that sent this ready event */ if (( msg.buf[0] != '\0' ) && ( strnlen( msg.buf, BUF_SIZE) <= MAX_CHARS_HOSTNAME )) { controller = msg.buf ; ilog ("%s Heartbeat Service Ready Event (%s)\n", msg.buf, sock_ptr->mtc_event_rx_sock->get_src_str()); } else { ilog ("Heartbeat Service Ready Event\n"); } obj_ptr->hbs_ready = true ; /* Run Maintenance on Inventory */ for ( temp = obj_ptr->hostname_inventory.begin () ; temp != obj_ptr->hostname_inventory.end () ; temp++ ) { string hostname = "" ; hostname.append( temp->c_str() ) ; /* Add all hosts, even the active controller, to * the heartbeat service. This tell the heartbeat * service about all the hosts so that it will * send heartbeat oob flag events to mtce. */ if ( send_hbs_command( hostname, MTC_CMD_ADD_HOST, controller ) != PASS ) { elog ("%s Failed to send inventory to heartbeat service\n", hostname.c_str()); } /* Send the start event to the heartbeat service for all enabled hosts */ if (( obj_ptr->get_adminState ( hostname ) == MTC_ADMIN_STATE__UNLOCKED ) && ( obj_ptr->get_operState ( hostname ) == MTC_OPER_STATE__ENABLED ) && ((obj_ptr->get_availStatus ( hostname ) == MTC_AVAIL_STATUS__AVAILABLE ) || (obj_ptr->get_availStatus ( hostname ) == MTC_AVAIL_STATUS__DEGRADED ))) { send_hbs_command ( hostname, MTC_CMD_START_HOST, controller ); } } } else { wlog ("Unrecognized Event from Heartbeat Service (hbsAgent)\n"); } return PASS ; } int send_hwmon_command ( string hostname, int command ) { int rc = PASS ; switch ( command ) { case MTC_CMD_QRY_HOST: { ilog ("%s sending 'sensor read' request\n", hostname.c_str()); break ; } case MTC_CMD_START_HOST: case MTC_CMD_STOP_HOST: case MTC_CMD_ADD_HOST: case MTC_CMD_MOD_HOST: case MTC_CMD_DEL_HOST: { if ( command == MTC_CMD_START_HOST ) { get_mtcInv_ptr()->set_hwmond_monitor_state ( hostname, true ); } else if ( command == MTC_CMD_STOP_HOST ) { get_mtcInv_ptr()->set_hwmond_monitor_state ( hostname, false ); } ilog ("%s sending '%s' to hwmond service\n", hostname.c_str(), get_event_str(command).c_str()); break ; } default: { slog ("%s Unsupported command operation 0x%x\n", hostname.c_str(), command ); rc = FAIL_BAD_PARM ; } } if ( rc == PASS ) { mtc_message_type cmd ; string hwmon_info = "" ; int bytes = 0; mtc_socket_type * sock_ptr = get_sockPtr (); nodeLinkClass * obj_ptr = get_mtcInv_ptr (); memset ( &cmd, 0 , sizeof(mtc_message_type)); snprintf ( &cmd.hdr[0] , MSG_HEADER_SIZE, "%s", get_cmd_req_msg_header()); snprintf ( &cmd.hdr[MSG_HEADER_SIZE], MAX_CHARS_HOSTNAME, "%s", hostname.data()); /* Store the command, get the board management info and copy it into the message buffer */ cmd.cmd = command ; hwmon_info = obj_ptr->get_hwmon_info ( hostname ); memcpy ( &cmd.buf[0], hwmon_info.data(), hwmon_info.length()); /* rc = message size */ bytes = sizeof(mtc_message_type); rc = sock_ptr->hwmon_cmd_sock->write((char *)&cmd, bytes, obj_ptr->my_float_ip.c_str(), 0); if ( 0 > rc ) { elog ("%s Failed sendto command to hwmond (%d:%s)\n", hostname.c_str(), errno, strerror(errno)); rc = FAIL_SOCKET_SENDTO ; } else { print_mtc_message ( hostname, MTC_CMD_TX, cmd, get_iface_name_str(MGMNT_INTERFACE), false ); rc = PASS ; } } return rc ; }