From 9e2ff8241103c1679ef7f6e5098bc95275db39fe Mon Sep 17 00:00:00 2001 From: "Kyale, Eliud" Date: Wed, 22 Mar 2023 10:06:37 -0400 Subject: [PATCH] Add failover state of peer to heartbeat msg - add failover state to heartbeat message ( 4 bits ) - add logic to survived_state to use peer's failover state to determine whether to exit survived state and enter normal state - throttle peer is normal events with a threshold of 10 used to ensure the peer is normal and stable - change fsm->send_event() log to debug from info log level - a few logging improvements; debug send_event logs - update copyright year 2023 Test plan: PASS - AIO-DX: iso install PASS - AIO-DX: crash the sm as indicated in bug and observe swact to standby PASS - AIO-DX: manual swact PASS - AIO-DX: power off active controller PASS - AIO-SX: install and basic sanity check PASS - AIO-SX: upgrade test to verify sm heartbeat messages changes still function when controllers are running different loads Closes-Bug: 2012519 Signed-off-by: Kyale, Eliud Change-Id: I1f86dcb8c9d9dbaf436b9240867f61adc405e88c --- service-mgmt/sm-common/src/sm_types.c | 191 +++++++++--------- service-mgmt/sm-common/src/sm_types.h | 93 +++++++-- service-mgmt/sm/src/sm_failover.c | 128 ++++++++---- service-mgmt/sm/src/sm_failover.h | 29 +-- .../sm/src/sm_failover_fail_pending_state.cpp | 4 +- .../sm/src/sm_failover_failed_state.cpp | 2 + service-mgmt/sm/src/sm_failover_fsm.cpp | 2 +- .../sm/src/sm_failover_initial_state.cpp | 3 +- .../sm/src/sm_failover_normal_state.cpp | 4 +- service-mgmt/sm/src/sm_failover_ss.c | 24 ++- service-mgmt/sm/src/sm_failover_ss.h | 2 +- .../sm/src/sm_failover_survived_state.cpp | 11 +- service-mgmt/sm/src/sm_heartbeat_msg.c | 41 ++-- service-mgmt/sm/src/sm_heartbeat_msg.h | 8 +- service-mgmt/sm/src/sm_heartbeat_thread.c | 10 +- 15 files changed, 329 insertions(+), 223 deletions(-) diff --git a/service-mgmt/sm-common/src/sm_types.c b/service-mgmt/sm-common/src/sm_types.c index eb9d4a4b..7187c031 100644 --- a/service-mgmt/sm-common/src/sm_types.c +++ b/service-mgmt/sm-common/src/sm_types.c @@ -21,7 +21,7 @@ typedef struct char str[SM_VALUE_STR_MAPPING_MAX_CHAR]; } SmValueStrMappingT; -static SmValueStrMappingT +static SmValueStrMappingT _sm_node_admin_state_mappings[SM_NODE_ADMIN_STATE_MAX] = { { SM_NODE_ADMIN_STATE_NIL, "nil" }, @@ -59,7 +59,7 @@ _sm_node_ready_state_mappings[SM_NODE_READY_STATE_MAX] = { SM_NODE_READY_STATE_DISABLED, "disabled" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_node_event_mappings[SM_NODE_EVENT_MAX] = { { SM_NODE_EVENT_NIL, "nil" }, @@ -69,7 +69,7 @@ _sm_node_event_mappings[SM_NODE_EVENT_MAX] = { SM_NODE_EVENT_AUDIT, "audit" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_interface_state_mappings[SM_INTERFACE_STATE_MAX] = { { SM_INTERFACE_STATE_NIL, "nil" }, @@ -79,7 +79,7 @@ _sm_interface_state_mappings[SM_INTERFACE_STATE_MAX] = { SM_INTERFACE_STATE_NOT_IN_USE, "not-in-use" } }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_network_type_mappings[SM_NETWORK_TYPE_MAX] = { { SM_NETWORK_TYPE_NIL, "nil" }, @@ -90,7 +90,7 @@ _sm_network_type_mappings[SM_NETWORK_TYPE_MAX] = { SM_NETWORK_TYPE_IPV6_UDP, "ipv6-udp" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_path_type_mappings[SM_PATH_TYPE_MAX] = { { SM_PATH_TYPE_NIL, "nil" }, @@ -100,7 +100,7 @@ _sm_path_type_mappings[SM_PATH_TYPE_MAX] = { SM_PATH_TYPE_STATUS_ONLY, "status-only" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_auth_type_mappings[SM_AUTH_TYPE_MAX] = { { SM_AUTH_TYPE_NIL, "nil" }, @@ -109,7 +109,7 @@ _sm_auth_type_mappings[SM_AUTH_TYPE_MAX] = { SM_AUTH_TYPE_HMAC_SHA512, "hmac-sha512" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_orchestration_type_mappings[SM_ORCHESTRATION_TYPE_MAX] = { { SM_ORCHESTRATION_TYPE_NIL, "nil" }, @@ -119,7 +119,7 @@ _sm_orchestration_type_mappings[SM_ORCHESTRATION_TYPE_MAX] = { SM_ORCHESTRATION_TYPE_HYBRID, "hybrid" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_designation_type_mappings[SM_DESIGNATION_TYPE_MAX] = { { SM_DESIGNATION_TYPE_NIL, "nil" }, @@ -129,7 +129,7 @@ _sm_designation_type_mappings[SM_DESIGNATION_TYPE_MAX] = { SM_DESIGNATION_TYPE_OTHER, "other" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_domain_state_mappings[SM_SERVICE_DOMAIN_STATE_MAX] = { { SM_SERVICE_DOMAIN_STATE_NIL, "nil" }, @@ -141,7 +141,7 @@ _sm_service_domain_state_mappings[SM_SERVICE_DOMAIN_STATE_MAX] = { SM_SERVICE_DOMAIN_STATE_OTHER, "other" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_domain_event_mappings[SM_SERVICE_DOMAIN_EVENT_MAX] = { { SM_SERVICE_DOMAIN_EVENT_NIL, "nil" }, @@ -154,7 +154,7 @@ _sm_service_domain_event_mappings[SM_SERVICE_DOMAIN_EVENT_MAX] = { SM_SERVICE_DOMAIN_EVENT_CHANGING_LEADER, "change-leader" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_domain_interface_event_mappings[SM_SERVICE_DOMAIN_INTERFACE_EVENT_MAX] = { { SM_SERVICE_DOMAIN_INTERFACE_EVENT_NIL, "nil" }, @@ -174,7 +174,8 @@ _sm_failover_event_mappings[SM_FAILOVER_EVENT_MAX] = {SM_FAILOVER_EVENT_IF_STATE_CHANGED, "interface-state-changed"}, {SM_FAILOVER_EVENT_FAIL_PENDING_TIMEOUT, "fail-pending-timeout"}, {SM_FAILOVER_EVENT_FAILED_RECOVERY_AUDIT, "failed-recovery-audit"}, - {SM_FAILOVER_EVENT_NODE_ENABLED, "node-enabled"} + {SM_FAILOVER_EVENT_NODE_ENABLED, "node-enabled"}, + {SM_FAILOVER_EVENT_PEER_IS_NORMAL, "peer-is-normal"} }; static SmValueStrMappingT @@ -196,7 +197,7 @@ _sm_failover_interface_state_mappings[SM_FAILOVER_INTERFACE_STATE_MAX] = {SM_FAILOVER_INTERFACE_DOWN, "down"} }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_domain_neighbor_state_mappings[SM_SERVICE_DOMAIN_NEIGHBOR_STATE_MAX] = { { SM_SERVICE_DOMAIN_NEIGHBOR_STATE_NIL, "nil" }, @@ -206,7 +207,7 @@ _sm_service_domain_neighbor_state_mappings[SM_SERVICE_DOMAIN_NEIGHBOR_STATE_MAX] { SM_SERVICE_DOMAIN_NEIGHBOR_STATE_FULL, "full" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_domain_neighbor_event_mappings[SM_SERVICE_DOMAIN_NEIGHBOR_EVENT_MAX] = { { SM_SERVICE_DOMAIN_NEIGHBOR_EVENT_NIL, "nil" }, @@ -219,7 +220,7 @@ _sm_service_domain_neighbor_event_mappings[SM_SERVICE_DOMAIN_NEIGHBOR_EVENT_MAX] { SM_SERVICE_DOMAIN_NEIGHBOR_EVENT_DOWN, "down" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_domain_member_redundancy_model_mappings[SM_SERVICE_DOMAIN_MEMBER_REDUNDANCY_MODEL_MAX] = { { SM_SERVICE_DOMAIN_MEMBER_REDUNDANCY_MODEL_NIL, "nil" }, @@ -231,7 +232,7 @@ _sm_service_domain_member_redundancy_model_mappings[SM_SERVICE_DOMAIN_MEMBER_RED { SM_SERVICE_DOMAIN_MEMBER_REDUNDANCY_MODEL_N_TO_N, "N to N" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_domain_split_brain_recovery_mappings[SM_SERVICE_DOMAIN_SPLIT_BRAIN_RECOVERY_MAX] = { { SM_SERVICE_DOMAIN_SPLIT_BRAIN_RECOVERY_NIL, "nil" }, @@ -240,7 +241,7 @@ _sm_service_domain_split_brain_recovery_mappings[SM_SERVICE_DOMAIN_SPLIT_BRAIN_R { SM_SERVICE_DOMAIN_SPLIT_BRAIN_RECOVERY_SELECT_BEST_ACTIVE, "select-best-active" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_group_action_mappings[SM_SERVICE_GROUP_ACTION_MAX] = { { SM_SERVICE_GROUP_ACTION_NIL, "nil" }, @@ -252,7 +253,7 @@ _sm_service_group_action_mappings[SM_SERVICE_GROUP_ACTION_MAX] = { SM_SERVICE_GROUP_ACTION_RECOVER, "recover" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_group_state_mappings[SM_SERVICE_GROUP_STATE_MAX] = { { SM_SERVICE_GROUP_STATE_NIL, "nil" }, @@ -268,7 +269,7 @@ _sm_service_group_state_mappings[SM_SERVICE_GROUP_STATE_MAX] = { SM_SERVICE_GROUP_STATE_SHUTDOWN, "shutdown" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_group_event_mappings[SM_SERVICE_GROUP_EVENT_MAX] = { { SM_SERVICE_GROUP_EVENT_NIL, "nil" }, @@ -284,7 +285,7 @@ _sm_service_group_event_mappings[SM_SERVICE_GROUP_EVENT_MAX] = { SM_SERVICE_GROUP_EVENT_NOTIFICATION_TIMEOUT, "notification-timeout" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_group_status_mappings[SM_SERVICE_GROUP_STATUS_MAX] = { { SM_SERVICE_GROUP_STATUS_NIL, "nil" }, @@ -295,7 +296,7 @@ _sm_service_group_status_mappings[SM_SERVICE_GROUP_STATUS_MAX] = { SM_SERVICE_GROUP_STATUS_FAILED, "failed" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_group_condition_mappings[SM_SERVICE_GROUP_CONDITION_MAX] = { { SM_SERVICE_GROUP_CONDITION_NIL, "nil" }, @@ -312,7 +313,7 @@ _sm_service_group_condition_mappings[SM_SERVICE_GROUP_CONDITION_MAX] = { SM_SERVICE_GROUP_CONDITION_FD_LIMIT_REACHED, "FD-limit-reached" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_group_notification_mappings[SM_SERVICE_GROUP_NOTIFICATION_MAX] = { { SM_SERVICE_GROUP_NOTIFICATION_NIL, "nil" }, @@ -335,7 +336,7 @@ _sm_service_admin_state_mappings[SM_SERVICE_ADMIN_STATE_MAX] = { SM_SERVICE_ADMIN_STATE_UNLOCKED, "unknown" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_state_mappings[SM_SERVICE_STATE_MAX] = { { SM_SERVICE_STATE_NIL, "nil" }, @@ -353,7 +354,7 @@ _sm_service_state_mappings[SM_SERVICE_STATE_MAX] = { SM_SERVICE_STATE_SHUTDOWN, "shutdown" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_event_mappings[SM_SERVICE_EVENT_MAX] = { { SM_SERVICE_EVENT_NIL, "nil" }, @@ -388,7 +389,7 @@ _sm_service_event_mappings[SM_SERVICE_EVENT_MAX] = { SM_SERVICE_EVENT_SHUTDOWN, "shutdown" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_status_mappings[SM_SERVICE_STATUS_MAX] = { { SM_SERVICE_STATUS_NIL, "nil" }, @@ -399,7 +400,7 @@ _sm_service_status_mappings[SM_SERVICE_STATUS_MAX] = { SM_SERVICE_STATUS_FAILED, "failed" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_condition_mappings[SM_SERVICE_CONDITION_MAX] = { { SM_SERVICE_CONDITION_NIL, "nil" }, @@ -418,7 +419,7 @@ _sm_service_condition_mappings[SM_SERVICE_CONDITION_MAX] = { SM_SERVICE_CONDITION_FD_LIMIT_REACHED, "FD-limit-reached" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_severity_mappings[SM_SERVICE_SEVERITY_MAX] = { { SM_SERVICE_SEVERITY_NIL, "nil" }, @@ -429,7 +430,7 @@ _sm_service_severity_mappings[SM_SERVICE_SEVERITY_MAX] = { SM_SERVICE_SEVERITY_CRITICAL, "critical" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_heartbeat_type_mappings[SM_SERVICE_HEARTBEAT_TYPE_MAX] = { { SM_SERVICE_HEARTBEAT_TYPE_NIL, "nil" }, @@ -438,7 +439,7 @@ _sm_service_heartbeat_type_mappings[SM_SERVICE_HEARTBEAT_TYPE_MAX] = { SM_SERVICE_HEARTBEAT_TYPE_UDP, "udp" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_heartbeat_state_mappings[SM_SERVICE_HEARTBEAT_STATE_MAX] = { { SM_SERVICE_HEARTBEAT_STATE_NIL, "nil" }, @@ -447,7 +448,7 @@ _sm_service_heartbeat_state_mappings[SM_SERVICE_HEARTBEAT_STATE_MAX] = { SM_SERVICE_HEARTBEAT_STATE_STOPPED, "stopped" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_dependency_type_mappings[SM_SERVICE_DEPENDENCY_TYPE_MAX] = { { SM_SERVICE_DEPENDENCY_TYPE_NIL, "nil" }, @@ -456,7 +457,7 @@ _sm_service_dependency_type_mappings[SM_SERVICE_DEPENDENCY_TYPE_MAX] = { SM_SERVICE_DEPENDENCY_TYPE_STATE, "state" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_action_mappings[SM_SERVICE_ACTION_MAX] = { { SM_SERVICE_ACTION_NIL, "nil" }, @@ -471,7 +472,7 @@ _sm_service_action_mappings[SM_SERVICE_ACTION_MAX] = { SM_SERVICE_ACTION_AUDIT_DISABLED, "audit-disabled" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_action_result_mappings[SM_SERVICE_ACTION_RESULT_MAX] = { { SM_SERVICE_ACTION_RESULT_NIL, "nil" }, @@ -482,7 +483,7 @@ _sm_service_action_result_mappings[SM_SERVICE_ACTION_RESULT_MAX] = { SM_SERVICE_ACTION_RESULT_TIMEOUT, "timeout" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_domain_scheduling_state_mappings[SM_SERVICE_DOMAIN_SCHEDULING_STATE_MAX] = { { SM_SERVICE_DOMAIN_SCHEDULING_STATE_NIL, "nil" }, @@ -494,7 +495,7 @@ _sm_service_domain_scheduling_state_mappings[SM_SERVICE_DOMAIN_SCHEDULING_STATE_ { SM_SERVICE_DOMAIN_SCHEDULING_STATE_DISABLE_FORCE, "disable-force" }, }; -static SmValueStrMappingT +static SmValueStrMappingT _sm_service_domain_scheduling_list_mappings[] = { { SM_SERVICE_DOMAIN_SCHEDULING_LIST_NIL, "nil" }, @@ -525,7 +526,7 @@ typedef struct char str[SM_VALUE_STR_MAPPING_MAX_CHAR]; } SmNodeStateStrMappingT; -static SmNodeStateStrMappingT +static SmNodeStateStrMappingT _sm_node_state_mappings[SM_NODE_ADMIN_STATE_MAX*SM_NODE_READY_STATE_MAX] = { { SM_NODE_ADMIN_STATE_NIL, SM_NODE_READY_STATE_NIL, "nil-nil" }, @@ -568,7 +569,7 @@ static SmValueStrMappingT // **************************************************************************** // Types - Mapping Get String // ========================== -static const char* sm_mapping_get_str( SmValueStrMappingT mappings[], +static const char* sm_mapping_get_str( SmValueStrMappingT mappings[], unsigned int num_mappings, int value ) { unsigned int map_i; @@ -608,7 +609,7 @@ static int sm_mapping_get_value( SmValueStrMappingT mappings[], // ======================================= SmNodeAdminStateT sm_node_admin_state_value( const char* state_str ) { - return( (SmNodeAdminStateT) + return( (SmNodeAdminStateT) sm_mapping_get_value( _sm_node_admin_state_mappings, SM_NODE_ADMIN_STATE_MAX, state_str ) ); @@ -620,7 +621,7 @@ SmNodeAdminStateT sm_node_admin_state_value( const char* state_str ) // ======================================== const char* sm_node_admin_state_str( SmNodeAdminStateT state ) { - return( sm_mapping_get_str( _sm_node_admin_state_mappings, + return( sm_mapping_get_str( _sm_node_admin_state_mappings, SM_NODE_ADMIN_STATE_MAX, state ) ); } @@ -631,7 +632,7 @@ const char* sm_node_admin_state_str( SmNodeAdminStateT state ) // ==================================== SmNodeOperationalStateT sm_node_oper_state_value( const char* state_str ) { - return( (SmNodeOperationalStateT) + return( (SmNodeOperationalStateT) sm_mapping_get_value( _sm_node_oper_state_mappings, SM_NODE_OPERATIONAL_STATE_MAX, state_str ) ); @@ -643,7 +644,7 @@ SmNodeOperationalStateT sm_node_oper_state_value( const char* state_str ) // ===================================== const char* sm_node_oper_state_str( SmNodeOperationalStateT state ) { - return( sm_mapping_get_str( _sm_node_oper_state_mappings, + return( sm_mapping_get_str( _sm_node_oper_state_mappings, SM_NODE_OPERATIONAL_STATE_MAX, state ) ); } @@ -654,7 +655,7 @@ const char* sm_node_oper_state_str( SmNodeOperationalStateT state ) // ====================================== SmNodeAvailStatusT sm_node_avail_status_value( const char* status_str ) { - return( (SmNodeAvailStatusT) + return( (SmNodeAvailStatusT) sm_mapping_get_value( _sm_node_avail_status_mappings, SM_NODE_AVAIL_STATUS_MAX, status_str ) ); @@ -666,7 +667,7 @@ SmNodeAvailStatusT sm_node_avail_status_value( const char* status_str ) // ======================================= const char* sm_node_avail_status_str( SmNodeAvailStatusT status ) { - return( sm_mapping_get_str( _sm_node_avail_status_mappings, + return( sm_mapping_get_str( _sm_node_avail_status_mappings, SM_NODE_AVAIL_STATUS_MAX, status ) ); } @@ -677,7 +678,7 @@ const char* sm_node_avail_status_str( SmNodeAvailStatusT status ) // ============================== SmNodeReadyStateT sm_node_ready_state_value( const char* state_str ) { - return( (SmNodeReadyStateT) + return( (SmNodeReadyStateT) sm_mapping_get_value( _sm_node_ready_state_mappings, SM_NODE_READY_STATE_MAX, state_str ) ); @@ -689,7 +690,7 @@ SmNodeReadyStateT sm_node_ready_state_value( const char* state_str ) // =============================== const char* sm_node_ready_state_str( SmNodeReadyStateT state ) { - return( sm_mapping_get_str( _sm_node_ready_state_mappings, + return( sm_mapping_get_str( _sm_node_ready_state_mappings, SM_NODE_READY_STATE_MAX, state ) ); } @@ -721,7 +722,7 @@ const char* sm_node_state_str( SmNodeAdminStateT admin_state, // ======================== SmNodeEventT sm_node_event_value( const char* event_str ) { - return( (SmNodeEventT) + return( (SmNodeEventT) sm_mapping_get_value( _sm_node_event_mappings, SM_NODE_EVENT_MAX, event_str ) ); } @@ -742,7 +743,7 @@ const char* sm_node_event_str( SmNodeEventT event ) // ============================= SmInterfaceStateT sm_interface_state_value( const char* state_str ) { - return( (SmInterfaceStateT) + return( (SmInterfaceStateT) sm_mapping_get_value( _sm_interface_state_mappings, SM_INTERFACE_STATE_MAX, state_str ) ); } @@ -763,7 +764,7 @@ const char* sm_interface_state_str( SmInterfaceStateT state ) // ========================== SmNetworkTypeT sm_network_type_value( const char* network_type_str ) { - return( (SmNetworkTypeT) + return( (SmNetworkTypeT) sm_mapping_get_value( _sm_network_type_mappings, SM_NETWORK_TYPE_MAX, network_type_str ) ); } @@ -784,7 +785,7 @@ const char* sm_network_type_str( SmNetworkTypeT network_type ) // ======================= SmPathTypeT sm_path_type_value( const char* path_type_str ) { - return( (SmPathTypeT) + return( (SmPathTypeT) sm_mapping_get_value( _sm_path_type_mappings, SM_PATH_TYPE_MAX, path_type_str ) ); } @@ -805,7 +806,7 @@ const char* sm_path_type_str( SmPathTypeT path_type ) // ================================= SmAuthTypeT sm_auth_type_value( const char* auth_type_str ) { - return( (SmAuthTypeT) + return( (SmAuthTypeT) sm_mapping_get_value( _sm_auth_type_mappings, SM_AUTH_TYPE_MAX, auth_type_str ) ); } @@ -824,12 +825,12 @@ const char* sm_auth_type_str( SmAuthTypeT auth_type ) // **************************************************************************** // Types - Orchestration Type Value // ================================ -SmOrchestrationTypeT sm_orchestration_type_value( +SmOrchestrationTypeT sm_orchestration_type_value( const char* orchestration_type_str ) { - return( (SmOrchestrationTypeT) + return( (SmOrchestrationTypeT) sm_mapping_get_value( _sm_orchestration_type_mappings, - SM_ORCHESTRATION_TYPE_MAX, + SM_ORCHESTRATION_TYPE_MAX, orchestration_type_str ) ); } // **************************************************************************** @@ -840,7 +841,7 @@ SmOrchestrationTypeT sm_orchestration_type_value( const char* sm_orchestration_type_str( SmOrchestrationTypeT orchestration_type ) { return( sm_mapping_get_str( _sm_orchestration_type_mappings, - SM_ORCHESTRATION_TYPE_MAX, + SM_ORCHESTRATION_TYPE_MAX, orchestration_type ) ); } // **************************************************************************** @@ -848,12 +849,12 @@ const char* sm_orchestration_type_str( SmOrchestrationTypeT orchestration_type ) // **************************************************************************** // Types - Designation Type Value // ============================== -SmDesignationTypeT sm_designation_type_value( +SmDesignationTypeT sm_designation_type_value( const char* designation_type_str ) { - return( (SmDesignationTypeT) + return( (SmDesignationTypeT) sm_mapping_get_value( _sm_designation_type_mappings, - SM_DESIGNATION_TYPE_MAX, + SM_DESIGNATION_TYPE_MAX, designation_type_str ) ); } // **************************************************************************** @@ -873,7 +874,7 @@ const char* sm_designation_type_str( SmDesignationTypeT designation_type ) // ================================== SmServiceDomainStateT sm_service_domain_state_value( const char* state_str ) { - return( (SmServiceDomainStateT) + return( (SmServiceDomainStateT) sm_mapping_get_value( _sm_service_domain_state_mappings, SM_SERVICE_DOMAIN_STATE_MAX, state_str ) ); } @@ -894,7 +895,7 @@ const char* sm_service_domain_state_str( SmServiceDomainStateT state ) // ================================== SmServiceDomainEventT sm_service_domain_event_value( const char* event_str ) { - return( (SmServiceDomainEventT) + return( (SmServiceDomainEventT) sm_mapping_get_value( _sm_service_domain_event_mappings, SM_SERVICE_DOMAIN_EVENT_MAX, event_str ) ); } @@ -916,9 +917,9 @@ const char* sm_service_domain_event_str( SmServiceDomainEventT event ) SmServiceDomainSchedulingStateT sm_service_domain_scheduling_state_value( const char* sched_state_str ) { - return( (SmServiceDomainSchedulingStateT) + return( (SmServiceDomainSchedulingStateT) sm_mapping_get_value( _sm_service_domain_scheduling_state_mappings, - SM_SERVICE_DOMAIN_SCHEDULING_STATE_MAX, + SM_SERVICE_DOMAIN_SCHEDULING_STATE_MAX, sched_state_str ) ); } // **************************************************************************** @@ -941,9 +942,9 @@ const char* sm_service_domain_scheduling_state_str( SmServiceDomainSchedulingListT sm_service_domain_scheduling_list_value( const char* sched_list_str ) { - return( (SmServiceDomainSchedulingListT) + return( (SmServiceDomainSchedulingListT) sm_mapping_get_value( _sm_service_domain_scheduling_list_mappings, - SM_SERVICE_DOMAIN_SCHEDULING_LIST_MAX, + SM_SERVICE_DOMAIN_SCHEDULING_LIST_MAX, sched_list_str ) ); } // **************************************************************************** @@ -966,7 +967,7 @@ const char* sm_service_domain_scheduling_list_str( SmServiceDomainInterfaceEventT sm_service_domain_interface_event_value( const char* event_str ) { - return( (SmServiceDomainInterfaceEventT) + return( (SmServiceDomainInterfaceEventT) sm_mapping_get_value( _sm_service_domain_interface_event_mappings, SM_SERVICE_DOMAIN_INTERFACE_EVENT_MAX, event_str ) ); @@ -1024,9 +1025,9 @@ const char* sm_failover_interface_state_str( SmFailoverInterfaceStateT state ) SmServiceDomainNeighborStateT sm_service_domain_neighbor_state_value( const char* neighbor_state_str ) { - return( (SmServiceDomainNeighborStateT) + return( (SmServiceDomainNeighborStateT) sm_mapping_get_value( _sm_service_domain_neighbor_state_mappings, - SM_SERVICE_DOMAIN_NEIGHBOR_STATE_MAX, + SM_SERVICE_DOMAIN_NEIGHBOR_STATE_MAX, neighbor_state_str ) ); } // **************************************************************************** @@ -1049,9 +1050,9 @@ const char* sm_service_domain_neighbor_state_str( SmServiceDomainNeighborEventT sm_service_domainneighbor_event_value( const char* neighbor_event_str ) { - return( (SmServiceDomainNeighborEventT) + return( (SmServiceDomainNeighborEventT) sm_mapping_get_value( _sm_service_domain_neighbor_event_mappings, - SM_SERVICE_DOMAIN_NEIGHBOR_EVENT_MAX, + SM_SERVICE_DOMAIN_NEIGHBOR_EVENT_MAX, neighbor_event_str ) ); } // **************************************************************************** @@ -1121,7 +1122,7 @@ const char* sm_service_domain_split_brain_recovery_str( // **************************************************************************** // Types - Service Group Action Value // ================================== -SmServiceGroupActionT sm_service_group_action_value( +SmServiceGroupActionT sm_service_group_action_value( const char* action_str ) { return( (SmServiceGroupActionT) @@ -1135,7 +1136,7 @@ SmServiceGroupActionT sm_service_group_action_value( // =================================== const char* sm_service_group_action_str( SmServiceGroupActionT action ) { - return( sm_mapping_get_str( _sm_service_group_action_mappings, + return( sm_mapping_get_str( _sm_service_group_action_mappings, SM_SERVICE_GROUP_ACTION_MAX, action ) ); } // **************************************************************************** @@ -1143,7 +1144,7 @@ const char* sm_service_group_action_str( SmServiceGroupActionT action ) // **************************************************************************** // Types - Service Group State Value // ================================= -SmServiceGroupStateT sm_service_group_state_value( +SmServiceGroupStateT sm_service_group_state_value( const char* state_str ) { return( (SmServiceGroupStateT) @@ -1157,7 +1158,7 @@ SmServiceGroupStateT sm_service_group_state_value( // ================================== const char* sm_service_group_state_str( SmServiceGroupStateT state ) { - return( sm_mapping_get_str( _sm_service_group_state_mappings, + return( sm_mapping_get_str( _sm_service_group_state_mappings, SM_SERVICE_GROUP_STATE_MAX, state ) ); } // **************************************************************************** @@ -1302,7 +1303,7 @@ SmServiceGroupEventT sm_service_group_event_value( const char* event_str ) // ================================== const char* sm_service_group_event_str( SmServiceGroupEventT event ) { - return( sm_mapping_get_str( _sm_service_group_event_mappings, + return( sm_mapping_get_str( _sm_service_group_event_mappings, SM_SERVICE_GROUP_EVENT_MAX, event ) ); } // **************************************************************************** @@ -1310,7 +1311,7 @@ const char* sm_service_group_event_str( SmServiceGroupEventT event ) // **************************************************************************** // Types - Service Group Status Value // ================================== -SmServiceGroupStatusT sm_service_group_status_value( +SmServiceGroupStatusT sm_service_group_status_value( const char* status_str ) { return( (SmServiceGroupStatusT) @@ -1324,7 +1325,7 @@ SmServiceGroupStatusT sm_service_group_status_value( // =================================== const char* sm_service_group_status_str( SmServiceGroupStatusT status ) { - return( sm_mapping_get_str( _sm_service_group_status_mappings, + return( sm_mapping_get_str( _sm_service_group_status_mappings, SM_SERVICE_GROUP_STATUS_MAX, status ) ); } // **************************************************************************** @@ -1332,7 +1333,7 @@ const char* sm_service_group_status_str( SmServiceGroupStatusT status ) // **************************************************************************** // Types - Service Group Condition Value // ===================================== -SmServiceGroupConditionT sm_service_group_condition_value( +SmServiceGroupConditionT sm_service_group_condition_value( const char* condition_str ) { return( (SmServiceGroupConditionT) @@ -1347,7 +1348,7 @@ SmServiceGroupConditionT sm_service_group_condition_value( // ====================================== const char* sm_service_group_condition_str( SmServiceGroupConditionT condition ) { - return( sm_mapping_get_str( _sm_service_group_condition_mappings, + return( sm_mapping_get_str( _sm_service_group_condition_mappings, SM_SERVICE_GROUP_CONDITION_MAX, condition ) ); } // **************************************************************************** @@ -1383,7 +1384,7 @@ const char* sm_service_group_notification_str( SmServiceAdminStateT sm_service_admin_state_value( const char* state_str ) { return( (SmServiceAdminStateT) - sm_mapping_get_value( _sm_service_admin_state_mappings, + sm_mapping_get_value( _sm_service_admin_state_mappings, SM_SERVICE_ADMIN_STATE_MAX, state_str ) ); } // **************************************************************************** @@ -1393,7 +1394,7 @@ SmServiceAdminStateT sm_service_admin_state_value( const char* state_str ) // ================================== const char* sm_service_admin_state_str( SmServiceAdminStateT state ) { - return( sm_mapping_get_str( _sm_service_admin_state_mappings, + return( sm_mapping_get_str( _sm_service_admin_state_mappings, SM_SERVICE_ADMIN_STATE_MAX, state ) ); } // **************************************************************************** @@ -1404,7 +1405,7 @@ const char* sm_service_admin_state_str( SmServiceAdminStateT state ) SmServiceStateT sm_service_state_value( const char* state_str ) { return( (SmServiceStateT) - sm_mapping_get_value( _sm_service_state_mappings, + sm_mapping_get_value( _sm_service_state_mappings, SM_SERVICE_STATE_MAX, state_str ) ); } // **************************************************************************** @@ -1414,7 +1415,7 @@ SmServiceStateT sm_service_state_value( const char* state_str ) // ============================ const char* sm_service_state_str( SmServiceStateT state ) { - return( sm_mapping_get_str( _sm_service_state_mappings, + return( sm_mapping_get_str( _sm_service_state_mappings, SM_SERVICE_STATE_MAX, state ) ); } // **************************************************************************** @@ -1571,7 +1572,7 @@ SmServiceStateT sm_service_state_lesser( SmServiceStateT state_a, SmServiceEventT sm_service_event_value( const char* event_str ) { return( (SmServiceEventT) - sm_mapping_get_value( _sm_service_event_mappings, + sm_mapping_get_value( _sm_service_event_mappings, SM_SERVICE_EVENT_MAX, event_str ) ); } // **************************************************************************** @@ -1581,7 +1582,7 @@ SmServiceEventT sm_service_event_value( const char* event_str ) // ============================ const char* sm_service_event_str( SmServiceEventT event ) { - return( sm_mapping_get_str( _sm_service_event_mappings, + return( sm_mapping_get_str( _sm_service_event_mappings, SM_SERVICE_EVENT_MAX, event ) ); } // **************************************************************************** @@ -1602,7 +1603,7 @@ SmServiceStatusT sm_service_status_value( const char* status_str ) // ============================= const char* sm_service_status_str( SmServiceStatusT status ) { - return( sm_mapping_get_str( _sm_service_status_mappings, + return( sm_mapping_get_str( _sm_service_status_mappings, SM_SERVICE_STATUS_MAX, status ) ); } // **************************************************************************** @@ -1623,7 +1624,7 @@ SmServiceConditionT sm_service_condition_value( const char* condition_str ) // ================================ const char* sm_service_condition_str( SmServiceConditionT condition ) { - return( sm_mapping_get_str( _sm_service_condition_mappings, + return( sm_mapping_get_str( _sm_service_condition_mappings, SM_SERVICE_CONDITION_MAX, condition ) ); } // **************************************************************************** @@ -1644,7 +1645,7 @@ SmServiceSeverityT sm_service_severity_value( const char* severity_str ) // =============================== const char* sm_service_severity_str( SmServiceSeverityT severity ) { - return( sm_mapping_get_str( _sm_service_severity_mappings, + return( sm_mapping_get_str( _sm_service_severity_mappings, SM_SERVICE_SEVERITY_MAX, severity ) ); } // **************************************************************************** @@ -1652,7 +1653,7 @@ const char* sm_service_severity_str( SmServiceSeverityT severity ) // **************************************************************************** // Types - Service Heartbeat Type Value // ==================================== -SmServiceHeartbeatTypeT sm_service_heartbeat_type_value( +SmServiceHeartbeatTypeT sm_service_heartbeat_type_value( const char* heartbeat_type_str ) { return( (SmServiceHeartbeatTypeT) @@ -1668,7 +1669,7 @@ SmServiceHeartbeatTypeT sm_service_heartbeat_type_value( const char* sm_service_heartbeat_type_str( SmServiceHeartbeatTypeT heartbeat_type ) { - return( sm_mapping_get_str( _sm_service_heartbeat_type_mappings, + return( sm_mapping_get_str( _sm_service_heartbeat_type_mappings, SM_SERVICE_HEARTBEAT_TYPE_MAX, heartbeat_type ) ); } @@ -1677,7 +1678,7 @@ const char* sm_service_heartbeat_type_str( // **************************************************************************** // Types - Service Heartbeat State Value // ===================================== -SmServiceHeartbeatStateT sm_service_heartbeat_state_value( +SmServiceHeartbeatStateT sm_service_heartbeat_state_value( const char* heartbeat_state_str ) { return( (SmServiceHeartbeatStateT) @@ -1693,7 +1694,7 @@ SmServiceHeartbeatStateT sm_service_heartbeat_state_value( const char* sm_service_heartbeat_state_str( SmServiceHeartbeatStateT heartbeat_state ) { - return( sm_mapping_get_str( _sm_service_heartbeat_state_mappings, + return( sm_mapping_get_str( _sm_service_heartbeat_state_mappings, SM_SERVICE_HEARTBEAT_STATE_MAX, heartbeat_state ) ); } @@ -1706,7 +1707,7 @@ SmServiceDependencyTypeT sm_service_dependency_type_value( const char* type_str ) { return( (SmServiceDependencyTypeT) - sm_mapping_get_value( _sm_service_dependency_type_mappings, + sm_mapping_get_value( _sm_service_dependency_type_mappings, SM_SERVICE_DEPENDENCY_TYPE_MAX, type_str ) ); } // **************************************************************************** @@ -1716,7 +1717,7 @@ SmServiceDependencyTypeT sm_service_dependency_type_value( // ====================================== const char* sm_service_dependency_type_str( SmServiceDependencyTypeT type ) { - return( sm_mapping_get_str( _sm_service_dependency_type_mappings, + return( sm_mapping_get_str( _sm_service_dependency_type_mappings, SM_SERVICE_DEPENDENCY_TYPE_MAX, type ) ); } // **************************************************************************** @@ -1737,7 +1738,7 @@ SmServiceActionT sm_service_action_value( const char* action_str ) // ============================= const char* sm_service_action_str( SmServiceActionT action ) { - return( sm_mapping_get_str( _sm_service_action_mappings, + return( sm_mapping_get_str( _sm_service_action_mappings, SM_SERVICE_ACTION_MAX, action ) ); } // **************************************************************************** @@ -1758,7 +1759,7 @@ SmServiceActionResultT sm_service_action_result_value( const char* result_str ) // ==================================== const char* sm_service_action_result_str( SmServiceActionResultT result ) { - return( sm_mapping_get_str( _sm_service_action_result_mappings, + return( sm_mapping_get_str( _sm_service_action_result_mappings, SM_SERVICE_ACTION_RESULT_MAX, result ) ); } // **************************************************************************** @@ -1782,7 +1783,7 @@ bool sm_network_address_value( const char address_str[], result = inet_pton( AF_INET, address_str, &(address->u.ipv4.sin) ); } else if(( SM_NETWORK_TYPE_IPV6 == address->type )|| - ( SM_NETWORK_TYPE_IPV6_UDP == address->type )) + ( SM_NETWORK_TYPE_IPV6_UDP == address->type )) { result = inet_pton( AF_INET6, address_str, &(address->u.ipv6.sin6) ); @@ -1804,7 +1805,7 @@ bool sm_network_address_value( const char address_str[], // **************************************************************************** // Types - Network Address String // ============================== -void sm_network_address_str( const SmNetworkAddressT* address, +void sm_network_address_str( const SmNetworkAddressT* address, char address_str[] ) { address_str[0] = '\0'; diff --git a/service-mgmt/sm-common/src/sm_types.h b/service-mgmt/sm-common/src/sm_types.h index 25717a9f..4a4cd175 100644 --- a/service-mgmt/sm-common/src/sm_types.h +++ b/service-mgmt/sm-common/src/sm_types.h @@ -271,7 +271,7 @@ typedef enum SM_SERVICE_DOMAIN_EVENT_MAX } SmServiceDomainEventT; -typedef enum +typedef enum { SM_SERVICE_DOMAIN_EVENT_DATA_MSG_NODE_NAME, SM_SERVICE_DOMAIN_EVENT_DATA_MSG_GENERATION, @@ -281,11 +281,11 @@ typedef enum } SmServiceDomainEventDataT; typedef enum{ - SM_FAILOVER_STATE_INITIAL, - SM_FAILOVER_STATE_NORMAL, - SM_FAILOVER_STATE_FAIL_PENDING, - SM_FAILOVER_STATE_FAILED, - SM_FAILOVER_STATE_SURVIVED, + SM_FAILOVER_STATE_INITIAL = 0, + SM_FAILOVER_STATE_NORMAL = 1, + SM_FAILOVER_STATE_FAIL_PENDING = 2, + SM_FAILOVER_STATE_FAILED = 3, + SM_FAILOVER_STATE_SURVIVED = 4, SM_FAILOVER_STATE_MAX }SmFailoverStateT; @@ -295,6 +295,7 @@ typedef enum{ SM_FAILOVER_EVENT_FAIL_PENDING_TIMEOUT, SM_FAILOVER_EVENT_FAILED_RECOVERY_AUDIT, SM_FAILOVER_EVENT_NODE_ENABLED, + SM_FAILOVER_EVENT_PEER_IS_NORMAL, SM_FAILOVER_EVENT_MAX }SmFailoverEventT; @@ -340,7 +341,7 @@ typedef enum SM_SERVICE_DOMAIN_NEIGHBOR_EVENT_MAX } SmServiceDomainNeighborEventT; -typedef enum +typedef enum { SM_SERVICE_DOMAIN_NEIGHBOR_EVENT_DATA_MSG_EXCHANGE_SEQ, SM_SERVICE_DOMAIN_NEIGHBOR_EVENT_DATA_MSG_MORE_MEMBERS, @@ -427,7 +428,7 @@ typedef enum SM_SERVICE_GROUP_EVENT_MAX, } SmServiceGroupEventT; -typedef enum +typedef enum { SM_SERVICE_GROUP_EVENT_DATA_SERVICE_NAME, SM_SERVICE_GROUP_EVENT_DATA_SERVICE_STATE, @@ -567,7 +568,7 @@ typedef enum // action-failure: a service action has failed. // fatal-failure: a fatal failure has occured. // FD-limit-reached: limit of open FD has reached. -typedef enum +typedef enum { SM_SERVICE_CONDITION_NIL, SM_SERVICE_CONDITION_UNKNOWN, @@ -706,7 +707,59 @@ typedef enum SM_ERROR_MAX } SmErrorT; -typedef uint32_t SmHeartbeatMsgIfStateT; +/** + * The SM_FAILOVER_STATE_MASK is used for the failover states + * lower bits 8 to 10 . + * The 'SmFailoverStateT' is stored directly in the 4 bits + * and can be accessed by right shifting >> 8 + * stores upto 15 states or 0b1111 with 4 bits + * + * The SM_FAILOVER_IF_STATE_MASK is used for the state of the 4 interfaces + * If any of these interfaces is down ( assuming its configured) then the bit + * flag will be set + * - cluster_host 0b0001 + * - management 0b0010 + * - oam 0b0100 + * - admin 0b1000 + * + * SM_FAILOVER_HEARTBEAT_ALIVE & SM_FAILOVER_HELLO_MSG_ALIVE are currently only used locally + * and are not shared in the heartbeat message between controllers + * + * SM_FAILOVER_NODE_INFO_MASK is combination mask that is used to specify the bits shared in + * the heartbeat message shared between controllers + * + * Example: 0x100 indicate all "configured" interfaces are up and failover state is 'normal' + * Example: 0x130 is an example of a local message + * - heartbeat and hello message alive and all "configured" interfaces up + * - failover state is normal + */ +#define SM_FAILOVER_STATE_MASK 0x00000F00 // 4 bits in use for failover states +#define SM_FAILOVER_IF_STATE_MASK 0x0000000F // 4 bits in use for interface states +#define SM_FAILOVER_NODE_INFO_MASK (SM_FAILOVER_STATE_MASK | SM_FAILOVER_IF_STATE_MASK) + +/* Extract the failover state from the node info flags */ +#define SM_FAILOVER_EXTRACT_STATE(x) (SmFailoverStateT)((x & SM_FAILOVER_STATE_MASK) >> 8) + +typedef enum +{ + // --- interface status bits -- + SM_FAILOVER_CLUSTER_HOST_DOWN = (0x1 << 0), // 1 + SM_FAILOVER_MGMT_DOWN = (0x1 << 1), // 2 + SM_FAILOVER_OAM_DOWN = (0x1 << 2), // 4 + SM_FAILOVER_ADMIN_DOWN = (0x1 << 3), // 8 + // ---- local status bits -- + SM_FAILOVER_HEARTBEAT_ALIVE = (0x1 << 4), // 16 + SM_FAILOVER_HELLO_MSG_ALIVE = (0x1 << 5), // 32 + // ---- failover state bits -- + SM_FAILOVER_STATE_BIT0 = (0X1 << 8), + SM_FAILOVER_STATE_BIT1 = (0X1 << 9), + SM_FAILOVER_STATE_BIT2 = (0X1 << 10), + SM_FAILOVER_STATE_BIT3 = (0X1 << 11), + // ---------------------------- + SM_FAILOVER_PEER_DISABLED = (0x1 << 14) // 0x4000 +} SmFailoverNodeStateBitFlagT; + +typedef uint32_t SmHeartbeatMsgNodeInfoT; typedef enum { @@ -862,7 +915,7 @@ extern const char* sm_auth_type_str( SmAuthTypeT auth_type ); // **************************************************************************** // Types - Orchestration Type Value // ================================ -extern SmOrchestrationTypeT sm_orchestration_type_value( +extern SmOrchestrationTypeT sm_orchestration_type_value( const char* orchestration_type_str ); // **************************************************************************** @@ -876,14 +929,14 @@ extern const char* sm_orchestration_type_str( // **************************************************************************** // Types - Designation Type Value // ============================== -extern SmDesignationTypeT sm_designation_type_value( +extern SmDesignationTypeT sm_designation_type_value( const char* designation_type_str ); // **************************************************************************** // **************************************************************************** // Types - Designation Type String // =============================== -extern const char* sm_designation_type_str( +extern const char* sm_designation_type_str( SmDesignationTypeT designation_type ); // **************************************************************************** @@ -1000,7 +1053,7 @@ extern const char* sm_service_domain_neighbor_event_str( // **************************************************************************** // Types - Service Domain Member Redundancy Model Value // ==================================================== -extern SmServiceDomainMemberRedundancyModelT +extern SmServiceDomainMemberRedundancyModelT sm_service_domain_member_redundancy_model_value( const char* model_str ); // **************************************************************************** @@ -1015,7 +1068,7 @@ extern const char* sm_service_domain_member_redundancy_model_str( // **************************************************************************** // Types - Service Domain Split Brain Recovery Value // ================================================= -extern SmServiceDomainSplitBrainRecoveryT +extern SmServiceDomainSplitBrainRecoveryT sm_service_domain_split_brain_recovery_value( const char* recovery_str ); // **************************************************************************** @@ -1030,7 +1083,7 @@ extern const char* sm_service_domain_split_brain_recovery_str( // **************************************************************************** // Types - Service Group Action Value // ================================== -extern SmServiceGroupActionT sm_service_group_action_value( +extern SmServiceGroupActionT sm_service_group_action_value( const char* action_str ); // **************************************************************************** @@ -1075,7 +1128,7 @@ extern const char* sm_service_group_event_str( SmServiceGroupEventT event ); // **************************************************************************** // Types - Service Group Status Value // ================================== -extern SmServiceGroupStatusT sm_service_group_status_value( +extern SmServiceGroupStatusT sm_service_group_status_value( const char* status_str ); // **************************************************************************** @@ -1088,7 +1141,7 @@ extern const char* sm_service_group_status_str( SmServiceGroupStatusT status ); // **************************************************************************** // Types - Service Group Condition Value // ===================================== -extern SmServiceGroupConditionT sm_service_group_condition_value( +extern SmServiceGroupConditionT sm_service_group_condition_value( const char* condition_str ); // **************************************************************************** @@ -1195,7 +1248,7 @@ extern const char* sm_service_severity_str( SmServiceSeverityT severity ); // **************************************************************************** // Types - Service Heartbeat Type Value // ==================================== -extern SmServiceHeartbeatTypeT sm_service_heartbeat_type_value( +extern SmServiceHeartbeatTypeT sm_service_heartbeat_type_value( const char* heartbeat_type_str ); // **************************************************************************** @@ -1209,7 +1262,7 @@ extern const char* sm_service_heartbeat_type_str( // **************************************************************************** // Types - Service Heartbeat State Value // ===================================== -extern SmServiceHeartbeatStateT sm_service_heartbeat_state_value( +extern SmServiceHeartbeatStateT sm_service_heartbeat_state_value( const char* heartbeat_state_str ); // **************************************************************************** diff --git a/service-mgmt/sm/src/sm_failover.c b/service-mgmt/sm/src/sm_failover.c index 4a8cc508..37e6f8a0 100644 --- a/service-mgmt/sm/src/sm_failover.c +++ b/service-mgmt/sm/src/sm_failover.c @@ -47,6 +47,8 @@ #define SM_FAILOVER_RECOVERY_INTERVAL_IN_SEC 100 #define SM_FAILOVER_INTERFACE_STATE_REPORT_INTERVAL_MS 20000 +#define SM_FAILOVER_PEER_IS_NORMAL_THRESHOLD 10 + const char* RESET_PEER_NOW = "/var/run/.sm_reset_peer"; typedef enum @@ -124,8 +126,8 @@ class SmFailoverInterfaceInfo SmTimerIdT failover_audit_timer_id; static char _host_name[SM_NODE_NAME_MAX_CHAR]; static char _peer_name[SM_NODE_NAME_MAX_CHAR]; -static SmHeartbeatMsgIfStateT _peer_if_state = 0; -static SmHeartbeatMsgIfStateT _peer_if_state_at_last_action = 0; +static SmHeartbeatMsgNodeInfoT _peer_node_info = 0; +static SmHeartbeatMsgNodeInfoT _peer_node_info_at_last_action = 0; static bool _retry = true; static bool _recheck = true; @@ -145,7 +147,7 @@ static SmDbHandleT* _sm_db_handle = NULL; static SmNodeScheduleStateT _host_state; static SmNodeScheduleStateT _host_state_at_last_action; // host state when action was taken last time static int64_t _node_comm_state = -1; -static int _prev_if_state_flag = -1; +static int _prev_node_info_flag = -1; time_t _last_if_state_ms = 0; static SmNodeScheduleStateT _prev_host_state= SM_NODE_STATE_UNKNOWN; @@ -472,22 +474,38 @@ SmErrorT sm_failover_degrade_clear(SmFailoverDegradeSourceT source) // **************************************************************************** // **************************************************************************** -// Failover - peer interface state update +// Failover - peer node info update // ================== -void sm_failover_if_state_update(const char node_name[], SmHeartbeatMsgIfStateT if_state) +void sm_failover_node_info_update(const char node_name[], SmHeartbeatMsgNodeInfoT node_info) { + static uint32_t num_of_peer_is_normal_events = 0; if( 0 == strcmp(node_name, _peer_name) ) { mutex_holder holder(&sm_failover_mutex); - if( _peer_if_state != if_state ) + SmFailoverStateT node_failover_state = SM_FAILOVER_EXTRACT_STATE(node_info); + if(node_failover_state == SM_FAILOVER_STATE_NORMAL) { - DPRINTFI("%s I/F state changed %d => %d", node_name, _peer_if_state, if_state); - _peer_if_state = if_state; + num_of_peer_is_normal_events++; + if(num_of_peer_is_normal_events > SM_FAILOVER_PEER_IS_NORMAL_THRESHOLD) + { + SmFailoverFSM::get_fsm().send_event(SM_FAILOVER_EVENT_PEER_IS_NORMAL, NULL); + num_of_peer_is_normal_events = 0; // reset count + } } - }else + else + { + num_of_peer_is_normal_events = 0; // reset count + } + + if( _peer_node_info != node_info ) + { + DPRINTFI("%s Node Info changed %#x => %#x", node_name, _peer_node_info, node_info); + _peer_node_info = node_info; + } + } else { - DPRINTFE("If state updated by unknown host %s", node_name); + DPRINTFE("Node Info (%#x) updated by unknown host %s", node_info, node_name); } } // **************************************************************************** @@ -523,25 +541,25 @@ bool is_admin_interface_configured() // **************************************************************************** // **************************************************************************** -// Failover - get interface state +// Failover - get host interface state flags // ================== -int sm_failover_get_if_state() +static uint32_t sm_failover_get_host_if_state_flags() { SmFailoverInterfaceStateT mgmt_state = _mgmt_interface_info->get_state(); SmFailoverInterfaceStateT oam_state = _oam_interface_info->get_state(); SmFailoverInterfaceStateT cluster_host_state; SmFailoverInterfaceStateT admin_state; - int if_state_flag = 0; + uint32_t if_state_flags = 0; if ( is_cluster_host_interface_configured() ) { cluster_host_state = _cluster_host_interface_info->get_state(); if( SM_FAILOVER_INTERFACE_OK == cluster_host_state ) { - if_state_flag |= SM_FAILOVER_HEARTBEAT_ALIVE; + if_state_flags |= SM_FAILOVER_HEARTBEAT_ALIVE; } else if ( SM_FAILOVER_INTERFACE_DOWN == cluster_host_state ) { - if_state_flag |= SM_FAILOVER_CLUSTER_HOST_DOWN; + if_state_flags |= SM_FAILOVER_CLUSTER_HOST_DOWN; } } @@ -550,55 +568,60 @@ int sm_failover_get_if_state() admin_state = _admin_interface_info->get_state(); if( SM_FAILOVER_INTERFACE_OK == admin_state ) { - if_state_flag |= SM_FAILOVER_HEARTBEAT_ALIVE; + if_state_flags |= SM_FAILOVER_HEARTBEAT_ALIVE; } else if ( SM_FAILOVER_INTERFACE_DOWN == admin_state ) { - if_state_flag |= SM_FAILOVER_ADMIN_DOWN; + if_state_flags |= SM_FAILOVER_ADMIN_DOWN; } } if( SM_FAILOVER_INTERFACE_OK == mgmt_state ) { - if_state_flag |= SM_FAILOVER_HEARTBEAT_ALIVE; + if_state_flags |= SM_FAILOVER_HEARTBEAT_ALIVE; } else if ( SM_FAILOVER_INTERFACE_DOWN == mgmt_state ) { - if_state_flag |= SM_FAILOVER_MGMT_DOWN; + if_state_flags |= SM_FAILOVER_MGMT_DOWN; } if( SM_FAILOVER_INTERFACE_OK == oam_state ) { - if_state_flag |= SM_FAILOVER_HEARTBEAT_ALIVE; + if_state_flags |= SM_FAILOVER_HEARTBEAT_ALIVE; } else if ( SM_FAILOVER_INTERFACE_DOWN == oam_state ) { - if_state_flag |= SM_FAILOVER_OAM_DOWN; + if_state_flags |= SM_FAILOVER_OAM_DOWN; } - return if_state_flag; + return if_state_flags; } // **************************************************************************** // **************************************************************************** -// Failover - get interface state +// Failover - get host node info - bit flags // ================== -SmHeartbeatMsgIfStateT sm_failover_if_state_get() +SmHeartbeatMsgNodeInfoT sm_failover_get_host_node_info_flags() { mutex_holder holder(&sm_failover_mutex); - int if_state_flag = sm_failover_get_if_state(); - return (if_state_flag & 0b0111); //the lower 3 bits i/f state flag + + uint32_t failover_state_bits = ((uint32_t)SmFailoverFSM::get_fsm().get_state() << 8 ); + failover_state_bits &= SM_FAILOVER_STATE_MASK; + + uint32_t if_state_flags = sm_failover_get_host_if_state_flags(); + if_state_flags &= SM_FAILOVER_IF_STATE_MASK; + + return (failover_state_bits | if_state_flags); } // **************************************************************************** // **************************************************************************** -// Failover - get peer node interface state +// Failover - get peer node info - bit flags // ================== -SmHeartbeatMsgIfStateT sm_failover_get_peer_if_state() +SmHeartbeatMsgNodeInfoT sm_failover_get_peer_node_info_flags() { mutex_holder holder(&sm_failover_mutex); - - return (_peer_if_state & 0b0111); //the lower 3 bits i/f state flag + return (_peer_node_info & SM_FAILOVER_NODE_INFO_MASK); } // **************************************************************************** @@ -1015,7 +1038,9 @@ void sm_failover_audit() { if ( _prev_host_state != _host_state ) { - DPRINTFD("Wait for scheduler to decided my role. host state = %d", _host_state); + DPRINTFD("Wait for scheduler to decided my role. host state = %s(%d)", + sm_node_schedule_state_str(_host_state), + _host_state); _prev_host_state = _host_state; } return; @@ -1023,12 +1048,14 @@ void sm_failover_audit() if ( _prev_host_state != _host_state ) { - DPRINTFI("host state is %d", _host_state); + DPRINTFI("host state is %s(%d)", + sm_node_schedule_state_str(_host_state), + _host_state); _prev_host_state = _host_state; } - int if_state_flag = sm_failover_get_if_state(); - if(if_state_flag & SM_FAILOVER_HEARTBEAT_ALIVE) + int node_info_flags = (int)sm_failover_get_host_node_info_flags(); + if(node_info_flags & SM_FAILOVER_HEARTBEAT_ALIVE) { _heartbeat_count ++; } @@ -1036,12 +1063,14 @@ void sm_failover_audit() { _heartbeat_count = 0; } - if( _prev_if_state_flag != if_state_flag) + if( _prev_node_info_flag != node_info_flags) { - DPRINTFI("Interface state flag %d", if_state_flag); + DPRINTFI("Interface state flags changed : %#x -> %#x", + _prev_node_info_flag, + node_info_flags); _last_if_state_ms = now_ms; - _prev_if_state_flag = if_state_flag; + _prev_node_info_flag = node_info_flags; } if(!peer_controller_enabled()) @@ -1082,7 +1111,7 @@ void sm_failover_audit() _if_state_changed = false; } - int64_t curr_node_state = if_state_flag; + int64_t curr_node_state = node_info_flags; if( _hello_msg_alive ) { @@ -1092,7 +1121,7 @@ void sm_failover_audit() if( !_retry && !_recheck && ( _node_comm_state == curr_node_state && _host_state == _host_state_at_last_action && - _peer_if_state == _peer_if_state_at_last_action )) + _peer_node_info == _peer_node_info_at_last_action )) { return; } @@ -1102,7 +1131,7 @@ void sm_failover_audit() _node_comm_state = curr_node_state; _host_state_at_last_action = _host_state; - _peer_if_state_at_last_action = _peer_if_state; + _peer_node_info_at_last_action = _peer_node_info; _log_nodes_state(); } @@ -1195,10 +1224,14 @@ void _log_nodes_state() sm_node_avail_status_str(peer.avail_status) ); } - DPRINTFI("Host state %d, I/F state %d, peer I/F state %d", + SmFailoverStateT peer_failover_state = SM_FAILOVER_EXTRACT_STATE(_peer_node_info); + DPRINTFI("Host state %s(%d), I/F state %#x, peer I/F state %#x, peer failover state %s(%d)", + sm_node_schedule_state_str(_host_state), _host_state, _node_comm_state, - _peer_if_state + _peer_node_info, + sm_failover_state_str(peer_failover_state), + peer_failover_state ); } // **************************************************************************** @@ -1527,7 +1560,7 @@ void dump_interfaces_state(FILE* fp) void dump_peer_if_state(FILE* fp) { - fprintf(fp, " Peer Interface state: %d\n", _peer_if_state); + fprintf(fp, " Peer Node info: %#x\n", _peer_node_info); } void dump_failover_fsm_state(FILE* fp) @@ -1536,6 +1569,13 @@ void dump_failover_fsm_state(FILE* fp) fprintf(fp, " Failover FSM state: %s\n", sm_failover_state_str(state)); } +void dump_peer_failover_fsm_state(FILE* fp) +{ + uint32_t peer_failover_bits = _peer_node_info & SM_FAILOVER_STATE_MASK; + SmFailoverStateT state = (SmFailoverStateT)( peer_failover_bits >> 8 ); + fprintf(fp, " Peer Failover FSM state: %s\n", sm_failover_state_str(state)); +} + // **************************************************************************** // Failover - dump state // ====================== @@ -1546,5 +1586,5 @@ void sm_failover_dump_state(FILE* fp) dump_failover_fsm_state(fp); dump_interfaces_state(fp); dump_peer_if_state(fp); + dump_peer_failover_fsm_state(fp); } -// **************************************************************************** diff --git a/service-mgmt/sm/src/sm_failover.h b/service-mgmt/sm/src/sm_failover.h index eccd7025..b29ee411 100644 --- a/service-mgmt/sm/src/sm_failover.h +++ b/service-mgmt/sm/src/sm_failover.h @@ -6,6 +6,7 @@ #ifndef __SM_FAILOVER_H__ #define __SM_FAILOVER_H__ #include +#include #include "sm_types.h" #include "sm_service_domain_interface_table.h" #include "sm_db_nodes.h" @@ -29,16 +30,6 @@ typedef enum SM_FAILOVER_DEGRADE_SOURCE_IF_DOWN = 2 }SmFailoverDegradeSourceT; -typedef enum -{ - SM_FAILOVER_CLUSTER_HOST_DOWN = 1, - SM_FAILOVER_MGMT_DOWN = 2, - SM_FAILOVER_OAM_DOWN = 4, - SM_FAILOVER_HEARTBEAT_ALIVE = 8, - SM_FAILOVER_HELLO_MSG_ALIVE = 16, - SM_FAILOVER_ADMIN_DOWN = 32, - SM_FAILOVER_PEER_DISABLED = 0x4000, -}SmFailoverCommFaultBitFlagT; // **************************************************************************** // initialize mutex @@ -105,14 +96,14 @@ extern void sm_failover_interface_up( const char* const interface_name ); // **************************************************************************** // **************************************************************************** -// Failover - interface state update for peer -extern void sm_failover_if_state_update(const char node_name[], - SmHeartbeatMsgIfStateT if_state); +// Failover - node info update for peer +extern void sm_failover_node_info_update(const char node_name[], + SmHeartbeatMsgNodeInfoT node_info); // **************************************************************************** // **************************************************************************** -// Failover - get local interface state -extern SmHeartbeatMsgIfStateT sm_failover_if_state_get(); +// Failover - get local interface state flags +extern SmHeartbeatMsgNodeInfoT sm_failover_get_host_node_info_flags(); // **************************************************************************** // **************************************************************************** @@ -147,9 +138,9 @@ SmFailoverInterfaceStateT sm_failover_get_interface_info(SmInterfaceTypeT interf // **************************************************************************** -// Failover - get peer node interface state +// Failover - get peer node info flags // ================== -SmHeartbeatMsgIfStateT sm_failover_get_peer_if_state(); +SmHeartbeatMsgNodeInfoT sm_failover_get_peer_node_info_flags(); // **************************************************************************** @@ -177,10 +168,6 @@ SmErrorT sm_failover_disable_peer(); bool sm_is_active_controller(); // **************************************************************************** -// **************************************************************************** -// Failover - get interface state -// ================== -int sm_failover_get_if_state(); // **************************************************************************** // **************************************************************************** diff --git a/service-mgmt/sm/src/sm_failover_fail_pending_state.cpp b/service-mgmt/sm/src/sm_failover_fail_pending_state.cpp index c80c10e6..efcb76b9 100644 --- a/service-mgmt/sm/src/sm_failover_fail_pending_state.cpp +++ b/service-mgmt/sm/src/sm_failover_fail_pending_state.cpp @@ -218,7 +218,7 @@ SmErrorT SmFailoverFailPendingState::event_handler(SmFailoverEventT event, const case SM_FAILOVER_EVENT_FAIL_PENDING_TIMEOUT: sm_node_utils_is_aio_duplex(&duplex); if( duplex && - 0 == (sm_failover_get_if_state() & SM_FAILOVER_HEARTBEAT_ALIVE)) + 0 == (sm_failover_get_host_node_info_flags() & SM_FAILOVER_HEARTBEAT_ALIVE)) { SmSystemModeT system_mode = sm_node_utils_get_system_mode(); SmInterfaceTypeT interfaces_to_check[3] = {SM_INTERFACE_UNKNOWN}; @@ -291,6 +291,8 @@ SmErrorT SmFailoverFailPendingState::event_handler(SmFailoverEventT event, const return error; } break; + case SM_FAILOVER_EVENT_PEER_IS_NORMAL: + break; default: DPRINTFE("Runtime error, unexpected event %s, at state %s", sm_failover_event_str(event), diff --git a/service-mgmt/sm/src/sm_failover_failed_state.cpp b/service-mgmt/sm/src/sm_failover_failed_state.cpp index 59821dd5..dc3cb3d3 100644 --- a/service-mgmt/sm/src/sm_failover_failed_state.cpp +++ b/service-mgmt/sm/src/sm_failover_failed_state.cpp @@ -282,6 +282,8 @@ SmErrorT SmFailoverFailedState::event_handler(SmFailoverEventT event, const ISmF } break; } + case SM_FAILOVER_EVENT_PEER_IS_NORMAL: + break; default: DPRINTFE("Runtime error, unexpected event %s, at state %s", sm_failover_event_str(event), diff --git a/service-mgmt/sm/src/sm_failover_fsm.cpp b/service-mgmt/sm/src/sm_failover_fsm.cpp index 07b815a8..e75b3747 100644 --- a/service-mgmt/sm/src/sm_failover_fsm.cpp +++ b/service-mgmt/sm/src/sm_failover_fsm.cpp @@ -109,7 +109,7 @@ SmErrorT SmFailoverFSM::send_event(SmFailoverEventT event, const ISmFSMEventData DPRINTFE("Runtime error. No handler for state %d", this->get_state()); return SM_FAILED; } - DPRINTFI("send_event %s\n", sm_failover_event_str(event)); + DPRINTFD("send_event %s\n", sm_failover_event_str(event)); state_handler->event_handler(event, event_data); return SM_OKAY; } diff --git a/service-mgmt/sm/src/sm_failover_initial_state.cpp b/service-mgmt/sm/src/sm_failover_initial_state.cpp index 9c205e36..01b34e33 100644 --- a/service-mgmt/sm/src/sm_failover_initial_state.cpp +++ b/service-mgmt/sm/src/sm_failover_initial_state.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2018 Wind River Systems, Inc. +// Copyright (c) 2018-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -18,6 +18,7 @@ SmErrorT SmFailoverInitialState::event_handler(SmFailoverEventT event, const ISm case SM_FAILOVER_EVENT_IF_STATE_CHANGED: case SM_FAILOVER_EVENT_FAIL_PENDING_TIMEOUT: case SM_FAILOVER_EVENT_NODE_ENABLED: + case SM_FAILOVER_EVENT_PEER_IS_NORMAL: break; default: DPRINTFE("Runtime error, unexpected event %s, at state %s", diff --git a/service-mgmt/sm/src/sm_failover_normal_state.cpp b/service-mgmt/sm/src/sm_failover_normal_state.cpp index 457a0496..f6a98fdd 100644 --- a/service-mgmt/sm/src/sm_failover_normal_state.cpp +++ b/service-mgmt/sm/src/sm_failover_normal_state.cpp @@ -43,7 +43,9 @@ SmErrorT SmFailoverNormalState::event_handler(SmFailoverEventT event, const ISmF } break; - + case SM_FAILOVER_EVENT_NODE_ENABLED: + case SM_FAILOVER_EVENT_PEER_IS_NORMAL: + break; default: DPRINTFE("Runtime error, unexpected event %s, at state %s", sm_failover_event_str(event), diff --git a/service-mgmt/sm/src/sm_failover_ss.c b/service-mgmt/sm/src/sm_failover_ss.c index 1ac7eb69..fe6ab06a 100644 --- a/service-mgmt/sm/src/sm_failover_ss.c +++ b/service-mgmt/sm/src/sm_failover_ss.c @@ -284,11 +284,11 @@ SmErrorT _get_system_status(SmSystemStatusT& sys_status, char host_name[], char SmSystemFailoverStatus::get_status().set_heartbeat_state(sys_status.heartbeat_state); sys_status.host_status.node_name = host_name; - sys_status.host_status.interface_state = sm_failover_if_state_get(); + sys_status.host_status.node_info_flags = sm_failover_get_host_node_info_flags(); sys_status.host_status.current_schedule_state = host_state; - sys_status.peer_status.node_name = peer_name; - sys_status.peer_status.interface_state = sm_failover_get_peer_if_state(); + sys_status.peer_status.node_name = peer_name; + sys_status.peer_status.node_info_flags = sm_failover_get_peer_node_info_flags(); sys_status.peer_status.current_schedule_state = sm_get_controller_state(peer_name); return SM_OKAY; } @@ -324,16 +324,18 @@ SmErrorT sm_failover_ss_get_survivor(const SmSystemStatusT& system_status, SmSys selection.set_peer_schedule_state(system_status.peer_status.current_schedule_state); if(SM_HEARTBEAT_OK == system_status.heartbeat_state) { - DPRINTFI("Heartbeat alive"); + DPRINTFI("Heartbeat alive; host flags %#x, peer flags %#x", + system_status.host_status.node_info_flags, + system_status.peer_status.node_info_flags); int host_healthy_score, peer_healthy_score; - host_healthy_score = get_node_if_healthy_score(system_status.host_status.interface_state); - peer_healthy_score = get_node_if_healthy_score(system_status.peer_status.interface_state); + host_healthy_score = get_node_if_healthy_score(system_status.host_status.node_info_flags); + peer_healthy_score = get_node_if_healthy_score(system_status.peer_status.node_info_flags); if( peer_healthy_score < host_healthy_score ) { //host is more healthy selection.set_host_schedule_state(SM_NODE_STATE_ACTIVE); selection.set_peer_schedule_state(SM_NODE_STATE_STANDBY); - if(system_status.peer_status.interface_state & SM_FAILOVER_MGMT_DOWN) + if(system_status.peer_status.node_info_flags & SM_FAILOVER_MGMT_DOWN) { DPRINTFI("Disable peer, host go active"); selection.set_peer_schedule_state(SM_NODE_STATE_FAILED); @@ -343,12 +345,18 @@ SmErrorT sm_failover_ss_get_survivor(const SmSystemStatusT& system_status, SmSys //peer is more healthy selection.set_host_schedule_state(SM_NODE_STATE_STANDBY); selection.set_peer_schedule_state(SM_NODE_STATE_ACTIVE); - if(system_status.host_status.interface_state & SM_FAILOVER_MGMT_DOWN) + if(system_status.host_status.node_info_flags & SM_FAILOVER_MGMT_DOWN) { DPRINTFI("Disable host, peer go active"); selection.set_host_schedule_state(SM_NODE_STATE_FAILED); } + } else + { + // host and peer has equal healthy scores + DPRINTFI("host and peer have the same healthy score"); } + DPRINTFI("host_healthy_score %d, peer_healthy_score %d", + host_healthy_score, peer_healthy_score); }else { DPRINTFI("Loss of heartbeat ALL"); diff --git a/service-mgmt/sm/src/sm_failover_ss.h b/service-mgmt/sm/src/sm_failover_ss.h index 2ca0f4f3..80dfdb66 100644 --- a/service-mgmt/sm/src/sm_failover_ss.h +++ b/service-mgmt/sm/src/sm_failover_ss.h @@ -13,7 +13,7 @@ typedef struct { const char* node_name; - unsigned int interface_state; + unsigned int node_info_flags; unsigned int heartbeat_state; SmFailoverInterfaceStateT mgmt_state; SmFailoverInterfaceStateT cluster_host_state; diff --git a/service-mgmt/sm/src/sm_failover_survived_state.cpp b/service-mgmt/sm/src/sm_failover_survived_state.cpp index d0e3b88a..ab3a93a7 100644 --- a/service-mgmt/sm/src/sm_failover_survived_state.cpp +++ b/service-mgmt/sm/src/sm_failover_survived_state.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2018 Wind River Systems, Inc. +// Copyright (c) 2018-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -9,6 +9,7 @@ #include "sm_debug.h" #include "sm_failover_fsm.h" #include "sm_failover_ss.h" +#include "sm_failover.h" static void _audit_failover_state() { @@ -44,6 +45,14 @@ SmErrorT SmFailoverSurvivedState::event_handler(SmFailoverEventT event, const IS { case SM_FAILOVER_EVENT_IF_STATE_CHANGED: break; + case SM_FAILOVER_EVENT_PEER_IS_NORMAL: + DPRINTFE("Peer is in %s(%d) failover state, Host will transition to %s(%d)", + sm_failover_state_str(SM_FAILOVER_STATE_NORMAL), + SM_FAILOVER_STATE_NORMAL, + sm_failover_state_str(SM_FAILOVER_STATE_NORMAL), + SM_FAILOVER_STATE_NORMAL); + SmFailoverFSM::get_fsm().set_state(SM_FAILOVER_STATE_NORMAL); + break; case SM_FAILOVER_EVENT_NODE_ENABLED: _audit_failover_state(); break; diff --git a/service-mgmt/sm/src/sm_heartbeat_msg.c b/service-mgmt/sm/src/sm_heartbeat_msg.c index 47afc1fc..9b453611 100644 --- a/service-mgmt/sm/src/sm_heartbeat_msg.c +++ b/service-mgmt/sm/src/sm_heartbeat_msg.c @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2017 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -55,7 +55,7 @@ typedef struct typedef struct { uint16_t msg_size; - uint32_t if_state; + uint32_t node_info; /**< @see SmHeartbeatMsgNodeInfoT */ } __attribute__ ((packed)) SmHeartbeatMsgAliveRev2T; typedef struct @@ -64,10 +64,10 @@ typedef struct union { - SmHeartbeatMsgAliveT alive; - SmHeartbeatMsgAliveRev2T if_state_msg; + SmHeartbeatMsgAliveT alive_msg; + SmHeartbeatMsgAliveRev2T aliveRev2_msg; char raw_msg[SM_HEARTBEAT_MSG_MAX_SIZE-sizeof(SmHeartbeatMsgHeaderT)]; - } u; + } msg; } __attribute__ ((packed)) SmHeartbeatMsgT; static char _tx_control_buffer[SM_HEARTBEAT_MSG_BUFFER_MAX_SIZE] __attribute__((aligned)); @@ -206,10 +206,10 @@ SmErrorT sm_heartbeat_msg_send_alive( SmNetworkTypeT network_type, char node_nam snprintf( heartbeat_msg.header.node_name, sizeof(heartbeat_msg.header.node_name), "%s", node_name ); - heartbeat_msg.u.if_state_msg.msg_size = htons((uint16_t)sizeof(SmHeartbeatMsgAliveRev2T)); - SmHeartbeatMsgIfStateT if_state; - if_state = sm_failover_if_state_get(); - heartbeat_msg.u.if_state_msg.if_state = htonl(if_state); + heartbeat_msg.msg.aliveRev2_msg.msg_size = htons((uint16_t)sizeof(SmHeartbeatMsgAliveRev2T)); + SmHeartbeatMsgNodeInfoT node_info; + node_info = sm_failover_get_host_node_info_flags(); + heartbeat_msg.msg.aliveRev2_msg.node_info = htonl(node_info); if( SM_AUTH_TYPE_HMAC_SHA512 == auth_type ) { @@ -269,7 +269,7 @@ SmErrorT sm_heartbeat_msg_send_alive( SmNetworkTypeT network_type, char node_nam // **************************************************************************** // **************************************************************************** -// Heartbeat Messaging - Get Ancillary Data +// Heartbeat Messaging - Get Ancillary Data // ======================================== static void* sm_heartbeat_msg_get_ancillary_data( struct msghdr* msg_hdr, int cmsg_level, int cmsg_type ) @@ -341,14 +341,14 @@ static void sm_heartbeat_msg_dispatch_msg( SmHeartbeatMsgT* heartbeat_msg, } uint16_t msg_size; - SmHeartbeatMsgIfStateT if_state = 0; + SmHeartbeatMsgNodeInfoT node_info = 0; bool perform_if_exg = false; if(2 <= ntohs(heartbeat_msg->header.revision)) { - msg_size = ntohs(heartbeat_msg->u.if_state_msg.msg_size); + msg_size = ntohs(heartbeat_msg->msg.aliveRev2_msg.msg_size); if(sizeof(SmHeartbeatMsgAliveRev2T) == msg_size ) { - if_state = ntohl(heartbeat_msg->u.if_state_msg.if_state); + node_info = ntohl(heartbeat_msg->msg.aliveRev2_msg.node_info); perform_if_exg = true; }else { @@ -377,14 +377,15 @@ static void sm_heartbeat_msg_dispatch_msg( SmHeartbeatMsgT* heartbeat_msg, if(perform_if_exg) { - if(NULL != callbacks->if_state) + if(NULL != callbacks->node_info) { - callbacks->if_state(heartbeat_msg->header.node_name, - if_state); + callbacks->node_info(heartbeat_msg->header.node_name, + node_info); } else { - DPRINTFD("No handler for if state package"); + DPRINTFD("Missing callback for node info %s %#x", heartbeat_msg->header.node_name, + node_info); } } } @@ -434,7 +435,7 @@ static void sm_heartbeat_msg_dispatch_ipv4_udp( int selobj, int64_t unused ) break; } else if( 0 == bytes_read ) { - // For connection oriented sockets, this indicates that the peer + // For connection oriented sockets, this indicates that the peer // has performed an orderly shutdown. return; @@ -453,7 +454,7 @@ static void sm_heartbeat_msg_dispatch_ipv4_udp( int selobj, int64_t unused ) char network_address_str[SM_NETWORK_ADDRESS_MAX_CHAR]; struct in_pktinfo* pkt_info; - pkt_info = (struct in_pktinfo*) + pkt_info = (struct in_pktinfo*) sm_heartbeat_msg_get_ancillary_data( &msg_hdr, SOL_IP, IP_PKTINFO ); if( NULL == pkt_info ) { @@ -666,7 +667,7 @@ static SmErrorT sm_heartbeat_msg_open_ipv4_udp_multicast_socket( // Bind socket to interface. memset(&ifr, 0, sizeof(ifr)); snprintf( ifr.ifr_name, sizeof(ifr.ifr_name), "%s", interface_name ); - + result = setsockopt( sock, SOL_SOCKET, SO_BINDTODEVICE, (void *)&ifr, sizeof(ifr) ); if( 0 > result ) diff --git a/service-mgmt/sm/src/sm_heartbeat_msg.h b/service-mgmt/sm/src/sm_heartbeat_msg.h index 1125d923..df75a292 100644 --- a/service-mgmt/sm/src/sm_heartbeat_msg.h +++ b/service-mgmt/sm/src/sm_heartbeat_msg.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2017 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -23,14 +23,14 @@ typedef void (*SmHeartbeatMsgAliveCallbackT) (char node_name[], SmNetworkAddressT* network_address, int network_port, int version, int revision, char interface_name[]); -typedef void (*SmHeartbeatMsgIfStateCallbackT) (const char node_name[], - SmHeartbeatMsgIfStateT if_state); +typedef void (*SmHeartbeatMsgNodeInfoCallbackT) (const char node_name[], + SmHeartbeatMsgNodeInfoT if_state); typedef struct { SmHeartbeatMsgAuthCallbackT auth; SmHeartbeatMsgAliveCallbackT alive; - SmHeartbeatMsgIfStateCallbackT if_state; + SmHeartbeatMsgNodeInfoCallbackT node_info; } SmHeartbeatMsgCallbacksT; // **************************************************************************** diff --git a/service-mgmt/sm/src/sm_heartbeat_thread.c b/service-mgmt/sm/src/sm_heartbeat_thread.c index cfb32fb3..c3c7f5ca 100644 --- a/service-mgmt/sm/src/sm_heartbeat_thread.c +++ b/service-mgmt/sm/src/sm_heartbeat_thread.c @@ -1116,12 +1116,12 @@ DONE: // **************************************************************************** // **************************************************************************** -// Heartbeat Thread - Receive if_state Message +// Heartbeat Thread - Receive node_info Message // ======================================== -static void sm_heartbeat_thread_receive_if_state_message( const char node_name[], - SmHeartbeatMsgIfStateT if_state) +static void sm_heartbeat_thread_receive_node_info_message( const char node_name[], + SmHeartbeatMsgNodeInfoT node_info) { - sm_failover_if_state_update(node_name, if_state); + sm_failover_node_info_update(node_name, node_info); } // **************************************************************************** @@ -1158,7 +1158,7 @@ static SmErrorT sm_heartbeat_thread_initialize_thread( void ) _callbacks.auth = sm_heartbeat_thread_auth_message; _callbacks.alive = sm_heartbeat_thread_receive_alive_message; - _callbacks.if_state = sm_heartbeat_thread_receive_if_state_message; + _callbacks.node_info = sm_heartbeat_thread_receive_node_info_message; error = sm_heartbeat_msg_register_callbacks( &_callbacks ); if( SM_OKAY != error )